arnaucube/arbo
1
1
Fork 0
mirror of https://github.com/arnaucube/arbo.git synced 2026-01-17 10:41:28 +01:00
Code Issues Projects Releases Wiki Activity

Implement addBatchInDisk for big trees

Browse Source 
Implement addBatchInDisk for big trees, which does not puts the tree in
memory, and works directly over the db data, parallelizing for each CPU.
This commit is contained in:
arnaucube
2021-10-14 16:59:26 +02:00
parent 3f7e769974
commit 6cf1e58d9f
10 changed files with 660 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vt.go
View File

@@ -37,7 +37,7 @@ type kv struct {
v []byte
}
func (p *params) keysValuesToKvs(ks, vs [][]byte) ([]kv, []Invalid, error) {
func keysValuesToKvs(maxLevels int, ks, vs [][]byte) ([]kv, []Invalid, error) {
if len(ks) != len(vs) {
return nil, nil, fmt.Errorf("len(keys)!=len(values) (%d!=%d)",
len(ks), len(vs))
@@ -45,7 +45,7 @@ func (p *params) keysValuesToKvs(ks, vs [][]byte) ([]kv, []Invalid, error) {
var invalids []Invalid
var kvs []kv
for i := 0; i < len(ks); i++ {
keyPath, err := keyPathFromKey(p.maxLevels, ks[i])
keyPath, err := keyPathFromKey(maxLevels, ks[i])
if err != nil {
invalids = append(invalids, Invalid{i, err})
continue
@@ -101,7 +101,7 @@ func (t *vt) addBatch(ks, vs [][]byte) ([]Invalid, error) {
l := int(math.Log2(float64(nCPU)))
kvs, invalids, err := t.params.keysValuesToKvs(ks, vs)
kvs, invalids, err := keysValuesToKvs(t.params.maxLevels, ks, vs)
if err != nil {
return invalids, err
}