You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

759 lines
19 KiB

  1. package arbo
  2. import (
  3. "bytes"
  4. "fmt"
  5. "math"
  6. "runtime"
  7. "sort"
  8. "sync"
  9. "github.com/iden3/go-merkletree/db"
  10. )
  11. /*
  12. AddBatch design
  13. ===============
  14. CASE A: Empty Tree --> if tree is empty (root==0)
  15. =================================================
  16. - Build the full tree from bottom to top (from all the leaf to the root)
  17. CASE B: ALMOST CASE A, Almost empty Tree --> if Tree has numLeafs < minLeafsThreshold
  18. ==============================================================================
  19. - Get the Leafs (key & value) (iterate the tree from the current root getting
  20. the leafs)
  21. - Create a new empty Tree
  22. - Do CASE A for the new Tree, giving the already existing key&values (leafs)
  23. from the original Tree + the new key&values to be added from the AddBatch call
  24. R R
  25. / \ / \
  26. A * / \
  27. / \ / \
  28. B C * *
  29. / | / \
  30. / | / \
  31. / | / \
  32. L: A B G D
  33. / \
  34. / \
  35. / \
  36. C *
  37. / \
  38. / \
  39. / \
  40. ... ... (nLeafs < minLeafsThreshold)
  41. CASE C: ALMOST CASE B --> if Tree has few Leafs (but numLeafs>=minLeafsThreshold)
  42. ==============================================================================
  43. - Use A, B, G, F as Roots of subtrees
  44. - Do CASE B for each subtree
  45. - Then go from L to the Root
  46. R
  47. / \
  48. / \
  49. / \
  50. * *
  51. / | / \
  52. / | / \
  53. / | / \
  54. L: A B G D
  55. / \
  56. / \
  57. / \
  58. C *
  59. / \
  60. / \
  61. / \
  62. ... ... (nLeafs >= minLeafsThreshold)
  63. CASE D: Already populated Tree
  64. ==============================
  65. - Use A, B, C, D as subtree
  66. - Sort the Keys in Buckets that share the initial part of the path
  67. - For each subtree add there the new leafs
  68. R
  69. / \
  70. / \
  71. / \
  72. * *
  73. / | / \
  74. / | / \
  75. / | / \
  76. L: A B C D
  77. /\ /\ / \ / \
  78. ... ... ... ... ... ...
  79. CASE E: Already populated Tree Unbalanced
  80. =========================================
  81. - Need to fill M1 and M2, and then will be able to use CASE D
  82. - Search for M1 & M2 in the inputed Keys
  83. - Add M1 & M2 to the Tree
  84. - From here can use CASE D
  85. R
  86. / \
  87. / \
  88. / \
  89. * *
  90. | \
  91. | \
  92. | \
  93. L: M1 * M2 * (where M1 and M2 are empty)
  94. / | /
  95. / | /
  96. / | /
  97. A * *
  98. / \ | \
  99. / \ | \
  100. / \ | \
  101. B * * C
  102. / \ |\
  103. ... ... | \
  104. | \
  105. D E
  106. Algorithm decision
  107. ==================
  108. - if nLeafs==0 (root==0): CASE A
  109. - if nLeafs<minLeafsThreshold: CASE B
  110. - if nLeafs>=minLeafsThreshold && (nLeafs/nBuckets) < minLeafsThreshold: CASE C
  111. - else: CASE D & CASE E
  112. - Multiple tree.Add calls: O(n log n)
  113. - Used in: cases A, B, C
  114. - Tree from bottom to top: O(log n)
  115. - Used in: cases D, E
  116. */
  117. const (
  118. minLeafsThreshold = 100 // nolint:gomnd // TMP WIP this will be autocalculated
  119. )
  120. // AddBatchOpt is the WIP implementation of the AddBatch method in a more
  121. // optimized approach.
  122. func (t *Tree) AddBatchOpt(keys, values [][]byte) ([]int, error) {
  123. t.updateAccessTime()
  124. t.Lock()
  125. defer t.Unlock()
  126. // when len(keyvalues) is not a power of 2, cut at the biggest power of
  127. // 2 under the len(keys), add those 2**n key-values using the AddBatch
  128. // approach, and then add the remaining key-values using tree.Add.
  129. kvs, err := t.keysValuesToKvs(keys, values)
  130. if err != nil {
  131. return nil, err
  132. }
  133. t.tx, err = t.db.NewTx()
  134. if err != nil {
  135. return nil, err
  136. }
  137. // if nCPU is not a power of two, cut at the highest power of two under
  138. // nCPU
  139. nCPU := highestPowerOfTwo(runtime.NumCPU())
  140. l := int(math.Log2(float64(nCPU)))
  141. var invalids []int
  142. // CASE A: if nLeafs==0 (root==0)
  143. if bytes.Equal(t.root, t.emptyHash) {
  144. invalids, err = t.caseA(nCPU, kvs)
  145. if err != nil {
  146. return nil, err
  147. }
  148. return t.finalizeAddBatch(len(keys), invalids)
  149. }
  150. // CASE B: if nLeafs<nBuckets
  151. nLeafs, err := t.GetNLeafs()
  152. if err != nil {
  153. return nil, err
  154. }
  155. if nLeafs < minLeafsThreshold { // CASE B
  156. var excedents []kv
  157. invalids, excedents, err = t.caseB(nCPU, 0, kvs)
  158. if err != nil {
  159. return nil, err
  160. }
  161. // add the excedents
  162. for i := 0; i < len(excedents); i++ {
  163. err = t.add(0, excedents[i].k, excedents[i].v)
  164. if err != nil {
  165. invalids = append(invalids, excedents[i].pos)
  166. }
  167. }
  168. return t.finalizeAddBatch(len(keys), invalids)
  169. }
  170. keysAtL, err := t.getKeysAtLevel(l + 1)
  171. if err != nil {
  172. return nil, err
  173. }
  174. // CASE C: if nLeafs>=minLeafsThreshold && (nLeafs/nBuckets) < minLeafsThreshold
  175. // available parallelization, will need to be a power of 2 (2**n)
  176. if nLeafs >= minLeafsThreshold &&
  177. (nLeafs/nCPU) < minLeafsThreshold &&
  178. len(keysAtL) == nCPU {
  179. invalids, err = t.caseC(nCPU, l, keysAtL, kvs)
  180. if err != nil {
  181. return nil, err
  182. }
  183. return t.finalizeAddBatch(len(keys), invalids)
  184. }
  185. // CASE E
  186. if len(keysAtL) != nCPU {
  187. // CASE E: add one key at each bucket, and then do CASE D
  188. buckets := splitInBuckets(kvs, nCPU)
  189. kvs = []kv{}
  190. for i := 0; i < len(buckets); i++ {
  191. // add one leaf of the bucket, if there is an error when
  192. // adding the k-v, try to add the next one of the bucket
  193. // (until one is added)
  194. var inserted int
  195. for j := 0; j < len(buckets[i]); j++ {
  196. if err := t.add(0, buckets[i][j].k, buckets[i][j].v); err == nil {
  197. inserted = j
  198. break
  199. }
  200. }
  201. // put the buckets elements except the inserted one
  202. kvs = append(kvs, buckets[i][:inserted]...)
  203. kvs = append(kvs, buckets[i][inserted+1:]...)
  204. }
  205. keysAtL, err = t.getKeysAtLevel(l + 1)
  206. if err != nil {
  207. return nil, err
  208. }
  209. }
  210. // CASE D
  211. if len(keysAtL) == nCPU { // enter in CASE D if len(keysAtL)=nCPU, if not, CASE E
  212. invalidsCaseD, err := t.caseD(nCPU, l, keysAtL, kvs)
  213. if err != nil {
  214. return nil, err
  215. }
  216. invalids = append(invalids, invalidsCaseD...)
  217. return t.finalizeAddBatch(len(keys), invalids)
  218. }
  219. return nil, fmt.Errorf("UNIMPLEMENTED")
  220. }
  221. func (t *Tree) finalizeAddBatch(nKeys int, invalids []int) ([]int, error) {
  222. // store root to db
  223. if err := t.tx.Put(dbKeyRoot, t.root); err != nil {
  224. return nil, err
  225. }
  226. // update nLeafs
  227. if err := t.incNLeafs(nKeys - len(invalids)); err != nil {
  228. return nil, err
  229. }
  230. // commit db tx
  231. if err := t.tx.Commit(); err != nil {
  232. return nil, err
  233. }
  234. return invalids, nil
  235. }
  236. func (t *Tree) caseA(nCPU int, kvs []kv) ([]int, error) {
  237. // if len(kvs) is not a power of 2, cut at the bigger power
  238. // of two under len(kvs), build the tree with that, and add
  239. // later the excedents
  240. kvsP2, kvsNonP2 := cutPowerOfTwo(kvs)
  241. invalids, err := t.buildTreeBottomUp(nCPU, kvsP2)
  242. if err != nil {
  243. return nil, err
  244. }
  245. for i := 0; i < len(kvsNonP2); i++ {
  246. if err = t.add(0, kvsNonP2[i].k, kvsNonP2[i].v); err != nil {
  247. invalids = append(invalids, kvsNonP2[i].pos)
  248. }
  249. }
  250. return invalids, nil
  251. }
  252. func (t *Tree) caseB(nCPU, l int, kvs []kv) ([]int, []kv, error) {
  253. // get already existing keys
  254. aKs, aVs, err := t.getLeafs(t.root)
  255. if err != nil {
  256. return nil, nil, err
  257. }
  258. aKvs, err := t.keysValuesToKvs(aKs, aVs)
  259. if err != nil {
  260. return nil, nil, err
  261. }
  262. // add already existing key-values to the inputted key-values
  263. // kvs = append(kvs, aKvs...)
  264. kvs, invalids := combineInKVSet(aKvs, kvs)
  265. // proceed with CASE A
  266. sortKvs(kvs)
  267. // cutPowerOfTwo, the excedent add it as normal Tree.Add
  268. kvsP2, kvsNonP2 := cutPowerOfTwo(kvs)
  269. var invalids2 []int
  270. if nCPU > 1 {
  271. invalids2, err = t.buildTreeBottomUp(nCPU, kvsP2)
  272. if err != nil {
  273. return nil, nil, err
  274. }
  275. } else {
  276. invalids2, err = t.buildTreeBottomUpSingleThread(kvsP2)
  277. if err != nil {
  278. return nil, nil, err
  279. }
  280. }
  281. invalids = append(invalids, invalids2...)
  282. // return the excedents which will be added at the full tree at the end
  283. return invalids, kvsNonP2, nil
  284. }
  285. func (t *Tree) caseC(nCPU, l int, keysAtL [][]byte, kvs []kv) ([]int, error) {
  286. // 1. go down until level L (L=log2(nBuckets)): keysAtL
  287. var excedents []kv
  288. buckets := splitInBuckets(kvs, nCPU)
  289. // 2. use keys at level L as roots of the subtrees under each one
  290. excedentsInBucket := make([][]kv, nCPU)
  291. subRoots := make([][]byte, nCPU)
  292. txs := make([]db.Tx, nCPU)
  293. var wg sync.WaitGroup
  294. wg.Add(nCPU)
  295. for i := 0; i < nCPU; i++ {
  296. go func(cpu int) {
  297. var err error
  298. txs[cpu], err = t.db.NewTx()
  299. if err != nil {
  300. panic(err) // TODO WIP
  301. }
  302. bucketTree := Tree{tx: txs[cpu], db: t.db, maxLevels: t.maxLevels,
  303. hashFunction: t.hashFunction, root: keysAtL[cpu]}
  304. // 3. do CASE B (with 1 cpu) for each key at level L
  305. _, bucketExcedents, err := bucketTree.caseB(1, l, buckets[cpu])
  306. if err != nil {
  307. panic(err) // TODO WIP
  308. // return nil, err
  309. }
  310. excedentsInBucket[cpu] = bucketExcedents
  311. subRoots[cpu] = bucketTree.root
  312. wg.Done()
  313. }(i)
  314. }
  315. wg.Wait()
  316. // merge buckets txs into Tree.tx
  317. for i := 0; i < len(txs); i++ {
  318. if err := t.tx.Add(txs[i]); err != nil {
  319. return nil, err
  320. }
  321. }
  322. for i := 0; i < len(excedentsInBucket); i++ {
  323. excedents = append(excedents, excedentsInBucket[i]...)
  324. }
  325. // 4. go upFromKeys from the new roots of the subtrees
  326. newRoot, err := t.upFromKeys(subRoots)
  327. if err != nil {
  328. return nil, err
  329. }
  330. t.root = newRoot
  331. // add the key-values that have not been used yet
  332. var invalids []int
  333. for i := 0; i < len(excedents); i++ {
  334. if err = t.add(0, excedents[i].k, excedents[i].v); err != nil {
  335. invalids = append(invalids, excedents[i].pos)
  336. }
  337. }
  338. return invalids, nil
  339. }
  340. func (t *Tree) caseD(nCPU, l int, keysAtL [][]byte, kvs []kv) ([]int, error) {
  341. if nCPU == 1 { // CASE D, but with 1 cpu
  342. var invalids []int
  343. for i := 0; i < len(kvs); i++ {
  344. if err := t.add(0, kvs[i].k, kvs[i].v); err != nil {
  345. invalids = append(invalids, kvs[i].pos)
  346. }
  347. }
  348. return invalids, nil
  349. }
  350. buckets := splitInBuckets(kvs, nCPU)
  351. subRoots := make([][]byte, nCPU)
  352. invalidsInBucket := make([][]int, nCPU)
  353. txs := make([]db.Tx, nCPU)
  354. var wg sync.WaitGroup
  355. wg.Add(nCPU)
  356. for i := 0; i < nCPU; i++ {
  357. go func(cpu int) {
  358. var err error
  359. txs[cpu], err = t.db.NewTx()
  360. if err != nil {
  361. panic(err) // TODO WIP
  362. }
  363. // put already existing tx into txs[cpu], as txs[cpu]
  364. // needs the pending key-values that are not in tree.db,
  365. // but are in tree.tx
  366. if err := txs[cpu].Add(t.tx); err != nil {
  367. panic(err) // TODO WIP
  368. }
  369. bucketTree := Tree{tx: txs[cpu], db: t.db, maxLevels: t.maxLevels - l,
  370. hashFunction: t.hashFunction, root: keysAtL[cpu]}
  371. for j := 0; j < len(buckets[cpu]); j++ {
  372. if err = bucketTree.add(l, buckets[cpu][j].k, buckets[cpu][j].v); err != nil {
  373. invalidsInBucket[cpu] = append(invalidsInBucket[cpu], buckets[cpu][j].pos)
  374. }
  375. }
  376. subRoots[cpu] = bucketTree.root
  377. wg.Done()
  378. }(i)
  379. }
  380. wg.Wait()
  381. // merge buckets txs into Tree.tx
  382. for i := 0; i < len(txs); i++ {
  383. if err := t.tx.Add(txs[i]); err != nil {
  384. return nil, err
  385. }
  386. }
  387. newRoot, err := t.upFromKeys(subRoots)
  388. if err != nil {
  389. return nil, err
  390. }
  391. t.root = newRoot
  392. var invalids []int
  393. for i := 0; i < len(invalidsInBucket); i++ {
  394. invalids = append(invalids, invalidsInBucket[i]...)
  395. }
  396. return invalids, nil
  397. }
  398. func splitInBuckets(kvs []kv, nBuckets int) [][]kv {
  399. buckets := make([][]kv, nBuckets)
  400. // 1. classify the keyvalues into buckets
  401. for i := 0; i < len(kvs); i++ {
  402. pair := kvs[i]
  403. // bucketnum := keyToBucket(pair.k, nBuckets)
  404. bucketnum := keyToBucket(pair.keyPath, nBuckets)
  405. buckets[bucketnum] = append(buckets[bucketnum], pair)
  406. }
  407. return buckets
  408. }
  409. // TODO rename in a more 'real' name (calculate bucket from/for key)
  410. func keyToBucket(k []byte, nBuckets int) int {
  411. nLevels := int(math.Log2(float64(nBuckets)))
  412. b := make([]int, nBuckets)
  413. for i := 0; i < nBuckets; i++ {
  414. b[i] = i
  415. }
  416. r := b
  417. mid := len(r) / 2 //nolint:gomnd
  418. for i := 0; i < nLevels; i++ {
  419. if int(k[i/8]&(1<<(i%8))) != 0 {
  420. r = r[mid:]
  421. mid = len(r) / 2 //nolint:gomnd
  422. } else {
  423. r = r[:mid]
  424. mid = len(r) / 2 //nolint:gomnd
  425. }
  426. }
  427. return r[0]
  428. }
  429. type kv struct {
  430. pos int // original position in the array
  431. keyPath []byte
  432. k []byte
  433. v []byte
  434. }
  435. // compareBytes compares byte slices where the bytes are compared from left to
  436. // right and each byte is compared by bit from right to left
  437. func compareBytes(a, b []byte) bool {
  438. // WIP
  439. for i := 0; i < len(a); i++ {
  440. for j := 0; j < 8; j++ {
  441. aBit := a[i] & (1 << j)
  442. bBit := b[i] & (1 << j)
  443. if aBit > bBit {
  444. return false
  445. } else if aBit < bBit {
  446. return true
  447. }
  448. }
  449. }
  450. return false
  451. }
  452. // sortKvs sorts the kv by path
  453. func sortKvs(kvs []kv) {
  454. sort.Slice(kvs, func(i, j int) bool {
  455. return compareBytes(kvs[i].keyPath, kvs[j].keyPath)
  456. })
  457. }
  458. func (t *Tree) keysValuesToKvs(ks, vs [][]byte) ([]kv, error) {
  459. if len(ks) != len(vs) {
  460. return nil, fmt.Errorf("len(keys)!=len(values) (%d!=%d)",
  461. len(ks), len(vs))
  462. }
  463. kvs := make([]kv, len(ks))
  464. for i := 0; i < len(ks); i++ {
  465. keyPath := make([]byte, t.hashFunction.Len())
  466. copy(keyPath[:], ks[i])
  467. kvs[i].pos = i
  468. kvs[i].keyPath = ks[i]
  469. kvs[i].k = ks[i]
  470. kvs[i].v = vs[i]
  471. }
  472. return kvs, nil
  473. }
  474. /*
  475. func (t *Tree) kvsToKeysValues(kvs []kv) ([][]byte, [][]byte) {
  476. ks := make([][]byte, len(kvs))
  477. vs := make([][]byte, len(kvs))
  478. for i := 0; i < len(kvs); i++ {
  479. ks[i] = kvs[i].k
  480. vs[i] = kvs[i].v
  481. }
  482. return ks, vs
  483. }
  484. */
  485. // buildTreeBottomUp splits the key-values into n Buckets (where n is the number
  486. // of CPUs), in parallel builds a subtree for each bucket, once all the subtrees
  487. // are built, uses the subtrees roots as keys for a new tree, which as result
  488. // will have the complete Tree build from bottom to up, where until the
  489. // log2(nCPU) level it has been computed in parallel.
  490. func (t *Tree) buildTreeBottomUp(nCPU int, kvs []kv) ([]int, error) {
  491. buckets := splitInBuckets(kvs, nCPU)
  492. subRoots := make([][]byte, nCPU)
  493. invalidsInBucket := make([][]int, nCPU)
  494. txs := make([]db.Tx, nCPU)
  495. var wg sync.WaitGroup
  496. wg.Add(nCPU)
  497. for i := 0; i < nCPU; i++ {
  498. go func(cpu int) {
  499. sortKvs(buckets[cpu])
  500. var err error
  501. txs[cpu], err = t.db.NewTx()
  502. if err != nil {
  503. panic(err) // TODO
  504. }
  505. bucketTree := Tree{tx: txs[cpu], db: t.db, maxLevels: t.maxLevels,
  506. hashFunction: t.hashFunction, root: t.emptyHash}
  507. currInvalids, err := bucketTree.buildTreeBottomUpSingleThread(buckets[cpu])
  508. if err != nil {
  509. panic(err) // TODO
  510. }
  511. invalidsInBucket[cpu] = currInvalids
  512. subRoots[cpu] = bucketTree.root
  513. wg.Done()
  514. }(i)
  515. }
  516. wg.Wait()
  517. // merge buckets txs into Tree.tx
  518. for i := 0; i < len(txs); i++ {
  519. if err := t.tx.Add(txs[i]); err != nil {
  520. return nil, err
  521. }
  522. }
  523. newRoot, err := t.upFromKeys(subRoots)
  524. if err != nil {
  525. return nil, err
  526. }
  527. t.root = newRoot
  528. var invalids []int
  529. for i := 0; i < len(invalidsInBucket); i++ {
  530. invalids = append(invalids, invalidsInBucket[i]...)
  531. }
  532. return invalids, err
  533. }
  534. // buildTreeBottomUpSingleThread builds the tree with the given []kv from bottom
  535. // to the root. keys & values must be sorted by path, and the array ks must be
  536. // length multiple of 2
  537. func (t *Tree) buildTreeBottomUpSingleThread(kvs []kv) ([]int, error) {
  538. // TODO check that log2(len(leafs)) < t.maxLevels, if not, maxLevels
  539. // would be reached and should return error
  540. var invalids []int
  541. // build the leafs
  542. leafKeys := make([][]byte, len(kvs))
  543. for i := 0; i < len(kvs); i++ {
  544. // TODO handle the case where Key&Value == 0
  545. leafKey, leafValue, err := newLeafValue(t.hashFunction, kvs[i].k, kvs[i].v)
  546. if err != nil {
  547. // return nil, err
  548. invalids = append(invalids, kvs[i].pos)
  549. }
  550. // store leafKey & leafValue to db
  551. if err := t.tx.Put(leafKey, leafValue); err != nil {
  552. // return nil, err
  553. invalids = append(invalids, kvs[i].pos)
  554. }
  555. leafKeys[i] = leafKey
  556. }
  557. r, err := t.upFromKeys(leafKeys)
  558. if err != nil {
  559. return invalids, err
  560. }
  561. t.root = r
  562. return invalids, nil
  563. }
  564. // keys & values must be sorted by path, and the array ks must be length
  565. // multiple of 2
  566. func (t *Tree) upFromKeys(ks [][]byte) ([]byte, error) {
  567. if len(ks) == 1 {
  568. return ks[0], nil
  569. }
  570. var rKs [][]byte
  571. for i := 0; i < len(ks); i += 2 {
  572. // TODO handle the case where Key&Value == 0
  573. k, v, err := newIntermediate(t.hashFunction, ks[i], ks[i+1])
  574. if err != nil {
  575. return nil, err
  576. }
  577. // store k-v to db
  578. if err = t.tx.Put(k, v); err != nil {
  579. return nil, err
  580. }
  581. rKs = append(rKs, k)
  582. }
  583. return t.upFromKeys(rKs)
  584. }
  585. func (t *Tree) getLeafs(root []byte) ([][]byte, [][]byte, error) {
  586. var ks, vs [][]byte
  587. err := t.iter(root, func(k, v []byte) {
  588. if v[0] != PrefixValueLeaf {
  589. return
  590. }
  591. leafK, leafV := readLeafValue(v)
  592. ks = append(ks, leafK)
  593. vs = append(vs, leafV)
  594. })
  595. return ks, vs, err
  596. }
  597. func (t *Tree) getKeysAtLevel(l int) ([][]byte, error) {
  598. var keys [][]byte
  599. err := t.iterWithStop(t.root, 0, func(currLvl int, k, v []byte) bool {
  600. if currLvl == l && !bytes.Equal(k, t.emptyHash) {
  601. keys = append(keys, k)
  602. }
  603. if currLvl >= l {
  604. return true // to stop the iter from going down
  605. }
  606. return false
  607. })
  608. return keys, err
  609. }
  610. // cutPowerOfTwo returns []kv of length that is a power of 2, and a second []kv
  611. // with the extra elements that don't fit in a power of 2 length
  612. func cutPowerOfTwo(kvs []kv) ([]kv, []kv) {
  613. x := len(kvs)
  614. if (x & (x - 1)) != 0 {
  615. p2 := highestPowerOfTwo(x)
  616. return kvs[:p2], kvs[p2:]
  617. }
  618. return kvs, nil
  619. }
  620. func highestPowerOfTwo(n int) int {
  621. res := 0
  622. for i := n; i >= 1; i-- {
  623. if (i & (i - 1)) == 0 {
  624. res = i
  625. break
  626. }
  627. }
  628. return res
  629. }
  630. // combineInKVSet combines two kv array in one single array without repeated
  631. // keys.
  632. func combineInKVSet(base, toAdd []kv) ([]kv, []int) {
  633. // TODO this is a naive version, this will be implemented in a more
  634. // efficient way or through maps, or through sorted binary search
  635. r := base
  636. var invalids []int
  637. for i := 0; i < len(toAdd); i++ {
  638. e := false
  639. // check if toAdd[i] exists in the base set
  640. for j := 0; j < len(base); j++ {
  641. if bytes.Equal(toAdd[i].k, base[j].k) {
  642. e = true
  643. }
  644. }
  645. if !e {
  646. r = append(r, toAdd[i])
  647. } else {
  648. invalids = append(invalids, toAdd[i].pos)
  649. }
  650. }
  651. return r, invalids
  652. }
  653. // func computeSimpleAddCost(nLeafs int) int {
  654. // // nLvls 2^nLvls
  655. // nLvls := int(math.Log2(float64(nLeafs)))
  656. // return nLvls * int(math.Pow(2, float64(nLvls)))
  657. // }
  658. //
  659. // func computeBottomUpAddCost(nLeafs int) int {
  660. // // 2^nLvls * 2 - 1
  661. // nLvls := int(math.Log2(float64(nLeafs)))
  662. // return (int(math.Pow(2, float64(nLvls))) * 2) - 1
  663. // }