arnaucube
arnaucube/goBlockchainDataAnalysisPublic Self-hosted repository
implemented generation of network map of address history, not finished
arnaucode idoctnef@openmailbox.org
Thu, 27 Jul 2017 13:57:05 +0200
commit

f9262f72a11182c75f708022929a28ebc8d4002a

parent

568bd6af4696103619f32242521db2a6253b5b6f

M exploreBlockchain.go → exploreBlockchain.go

@@ -3,13 +3,15 @@

import ( "fmt" + "gopkg.in/mgo.v2/bson" + "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcrpcclient" + "github.com/fatih/color" ) func explore(client *btcrpcclient.Client, blockHash string) { var realBlocks int - var nOrigin NodeModel nOrigin.Id = "origin" nOrigin.Label = "origin"

@@ -26,116 +28,122 @@ check(err)

block, err := client.GetBlockVerbose(bh) check(err) - var newBlock BlockModel - newBlock.Hash = block.Hash - newBlock.Height = block.Height - newBlock.Confirmations = block.Confirmations - - //get Fee value - th, err := chainhash.NewHashFromStr(block.Tx[0]) - check(err) - tx, err := client.GetRawTransactionVerbose(th) - check(err) - var totalFee float64 - for _, Vo := range tx.Vout { - totalFee = totalFee + Vo.Value - } - newBlock.Fee = totalFee - - //for each Tx, get the Tx value - var totalAmount float64 - /*inside each block, there are []Tx - inside each Tx, if is the Tx[0], is the Fees - in the Tx[n] where n>0, each Tx is independent, - and inside each Tx there are []Vout. - Usually Vout[0] is the real Tx value - and the Vout[1] is the rest of the amount in the original wallet. - Each Tx moves all the wallet amount, and the realTx amount is sent to the destination - and the rest of the wallet amount, is send to another owned wallet - */ //if len(block.Tx) < 10 { for k, txHash := range block.Tx { - //first Tx is the Fee - //after first Tx is the Sent Amount if k > 0 { + realBlocks++ + fmt.Print("Block Height: ") + fmt.Print(block.Height) + fmt.Print(", num of Tx: ") + fmt.Print(k) + fmt.Print("/") + fmt.Println(len(block.Tx) - 1) + th, err := chainhash.NewHashFromStr(txHash) check(err) tx, err := client.GetRawTransactionVerbose(th) check(err) - var originAddress string + + var originAddresses []string + var outputAddresses []string for _, Vi := range tx.Vin { th, err := chainhash.NewHashFromStr(Vi.Txid) check(err) txVi, err := client.GetRawTransactionVerbose(th) check(err) - if len(txVi.Vout[0].ScriptPubKey.Addresses) > 0 { - originAddress = txVi.Vout[0].ScriptPubKey.Addresses[0] + if len(txVi.Vout[Vi.Vout].ScriptPubKey.Addresses) > 0 { + for _, originAddr := range txVi.Vout[Vi.Vout].ScriptPubKey.Addresses { + originAddresses = append(originAddresses, originAddr) + var n1 NodeModel + n1.Id = originAddr + n1.Label = originAddr + n1.Title = originAddr + n1.Group = string(block.Height) + n1.Value = 1 + n1.Shape = "dot" + saveNode(nodeCollection, n1) + } } else { - originAddress = "origin" + originAddresses = append(originAddresses, "origin") } } for _, Vo := range tx.Vout { - totalAmount = totalAmount + Vo.Value + //if Vo.Value > 0 { + for _, outputAddr := range Vo.ScriptPubKey.Addresses { + outputAddresses = append(outputAddresses, outputAddr) + var n2 NodeModel + n2.Id = outputAddr + n2.Label = outputAddr + n2.Title = outputAddr + n2.Group = string(block.Height) + n2.Value = 1 + n2.Shape = "dot" + saveNode(nodeCollection, n2) - var blockTx TxModel - blockTx.Txid = tx.Txid - blockTx.Amount = Vo.Value - blockTx.From = originAddress - blockTx.To = Vo.ScriptPubKey.Addresses[0] - newBlock.Tx = append(newBlock.Tx, blockTx) + for _, originAddr := range originAddresses { + var e EdgeModel + e.From = originAddr + e.To = outputAddr + e.Label = Vo.Value + e.Txid = tx.Txid + e.Arrows = "to" + e.BlockHeight = block.Height + saveEdge(edgeCollection, e) + //fmt.Println(e) + } + } + //} } + fmt.Print("originAddresses: ") + fmt.Println(len(originAddresses)) + fmt.Print("outputAddresses: ") + fmt.Println(len(outputAddresses)) } } - - if totalAmount > 0 { - newBlock.Amount = totalAmount - saveBlock(blockCollection, newBlock) - fmt.Print("Height: ") - fmt.Println(newBlock.Height) - fmt.Print("Amount: ") - fmt.Println(newBlock.Amount) - fmt.Print("Fee: ") - fmt.Println(newBlock.Fee) - fmt.Println("-----") - realBlocks++ - } //} - //set the next block blockHash = block.NextHash - - //analyze block creator - for _, t := range newBlock.Tx { - var n1 NodeModel - var n2 NodeModel - n1.Id = t.From - n1.Label = t.From - n1.Title = t.From - n1.Group = newBlock.Hash - n1.Value = 1 - n1.Shape = "dot" - n2.Id = t.To - n2.Label = t.To - n2.Title = t.To - n2.Group = newBlock.Hash - n2.Value = 1 - n2.Shape = "dot" + } - var e EdgeModel - e.From = t.From - e.To = t.To - e.Label = t.Amount - e.Txid = t.Txid - e.Arrows = "to" + fmt.Print("realBlocks (blocks with Fee and Amount values): ") + fmt.Println(realBlocks) + fmt.Println("reached the end of blockchain") +} +func addressTree(address string) NetworkModel { + var network NetworkModel - saveNode(nodeCollection, n1) - saveNode(nodeCollection, n2) - saveEdge(edgeCollection, e) + var currentEdge EdgeModel + currentEdge.From = "a" + currentEdge.To = "b" + for currentEdge.From != currentEdge.To { + color.Green("for") + fmt.Println(address) + //get edges before the address + edges := []EdgeModel{} + err := edgeCollection.Find(bson.M{"to": address}).All(&edges) + check(err) + for _, edge := range edges { + network.Edges = append(network.Edges, edge) + fmt.Println(edge) } + //get all nodes from edges + for _, edge := range edges { + node := NodeModel{} + err := nodeCollection.Find(bson.M{"id": edge.From}).One(&node) + check(err) + if nodeInNodes(network.Nodes, node) == false { + network.Nodes = append(network.Nodes, node) + } + err = nodeCollection.Find(bson.M{"id": edge.To}).One(&node) + check(err) + if nodeInNodes(network.Nodes, node) == false { + network.Nodes = append(network.Nodes, node) + } + } + address = edges[0].From + currentEdge = edges[0] } - fmt.Print("realBlocks (blocks with Fee and Amount values): ") - fmt.Println(realBlocks) - fmt.Println("reached the end of blockchain") + return network }
M main.go → main.go

@@ -55,6 +55,10 @@ if os.Args[1] == "-explore" {

color.Blue("starting to explore blockchain") explore(client, config.GenesisBlock) } + /*if os.Args[1] == "-tree" { + color.Blue("starting to make tree") + addressTree(client, "fY3HZxu7HFKRcYzVSTXRZpAJMP4qba2oR6") + }*/ } // Get the current block count.
M mongoModels.go → mongoModels.go

@@ -25,11 +25,12 @@ Shape string `json:"shape"`

} type EdgeModel struct { - Txid string `json:"txid"` - From string `json:"from"` - To string `json:"to"` - Label float64 `json:"label"` - Arrows string `json:"arrows"` + Txid string `json:"txid"` + From string `json:"from"` + To string `json:"to"` + Label float64 `json:"label"` + Arrows string `json:"arrows"` + BlockHeight int64 `json:"blockheight"` } type NetworkModel struct { Nodes []NodeModel `json:"nodes"`
M mongoOperations.go → mongoOperations.go

@@ -66,7 +66,7 @@ }

func getAllNodes() ([]NodeModel, error) { result := []NodeModel{} - iter := nodeCollection.Find(bson.M{}).Limit(100).Iter() + iter := nodeCollection.Find(bson.M{}).Limit(10000).Iter() err := iter.All(&result) return result, err }

@@ -74,13 +74,13 @@

func saveNode(c *mgo.Collection, node NodeModel) { //first, check if the node already exists result := NodeModel{} - err := c.Find(bson.M{"id": node.Id}).One(&result) + err := c.Find(bson.M{"id": node.Id, "group": node.Group}).One(&result) if err != nil { //node not found, so let's add a new entry err = c.Insert(node) check(err) } else { - err = c.Update(bson.M{"id": node.Id}, &node) + err = c.Update(bson.M{"id": node.Id, "group": node.Group}, &node) if err != nil { log.Fatal(err) }

@@ -89,22 +89,31 @@ }

func getAllEdges() ([]EdgeModel, error) { result := []EdgeModel{} - iter := edgeCollection.Find(bson.M{}).Limit(100).Iter() + iter := edgeCollection.Find(bson.M{}).Limit(10000).Iter() err := iter.All(&result) return result, err } func saveEdge(c *mgo.Collection, edge EdgeModel) { //first, check if the edge already exists result := EdgeModel{} - err := c.Find(bson.M{"txid": edge.Txid}).One(&result) + err := c.Find(bson.M{"txid": edge.Txid, "to": edge.To, "from": edge.From, "blockheight": edge.BlockHeight, "label": edge.Label}).One(&result) if err != nil { //edge not found, so let's add a new entry err = c.Insert(edge) check(err) } else { - err = c.Update(bson.M{"txid": edge.Txid}, &edge) + err = c.Update(bson.M{"txid": edge.Txid, "to": edge.To, "from": edge.From, "blockheight": edge.BlockHeight, "label": edge.Label}, &edge) if err != nil { log.Fatal(err) } } } + +func nodeInNodes(nodes []NodeModel, node NodeModel) bool { + for _, n := range nodes { + if n.Id == node.Id { + return true + } + } + return false +}
M serverRoutes.go → serverRoutes.go

@@ -4,6 +4,10 @@ import (

"encoding/json" "fmt" "net/http" + + "gopkg.in/mgo.v2/bson" + + "github.com/gorilla/mux" ) type Routes []Route

@@ -15,13 +19,18 @@ "GET",

"/", Index, }, - /* Route{ - "Recommendations", - "GET", - "/r/{userid}/{nrec}", - Recommendations, - }, - */ + Route{ + "AllAddresses", + "Get", + "/alladdresses", + AllAddresses, + }, + Route{ + "AddressNetwork", + "GET", + "/address/network/{address}", + AddressNetwork, + }, Route{ "NetworkMap", "Get",

@@ -60,6 +69,36 @@ fmt.Println(newUser)

fmt.Fprintln(w, "new user added: ", newUser.ID) } */ +func AllAddresses(w http.ResponseWriter, r *http.Request) { + ipFilter(w, r) + + nodes := []NodeModel{} + iter := nodeCollection.Find(bson.M{}).Limit(10000).Iter() + err := iter.All(&nodes) + + //convert []resp struct to json + jsonNodes, err := json.Marshal(nodes) + check(err) + + fmt.Fprintln(w, string(jsonNodes)) +} +func AddressNetwork(w http.ResponseWriter, r *http.Request) { + ipFilter(w, r) + + vars := mux.Vars(r) + address := vars["address"] + if address == "undefined" { + fmt.Fprintln(w, "not valid address") + } else { + network := addressTree(address) + + //convert []resp struct to json + jNetwork, err := json.Marshal(network) + check(err) + + fmt.Fprintln(w, string(jNetwork)) + } +} func NetworkMap(w http.ResponseWriter, r *http.Request) { ipFilter(w, r)
M web/app.js → web/app.js

@@ -11,6 +11,7 @@ 'angularBootstrapMaterial',

'app.navbar', 'app.main', 'app.network', + 'app.addressNetwork', 'app.sankey' ]). config(['$locationProvider', '$routeProvider', function($locationProvider, $routeProvider) {
M web/index.html → web/index.html

@@ -21,7 +21,7 @@ <link href="css/ionic.app.css" rel="stylesheet">

--> <style type="text/css"> #mynetwork { - background: black; + /*background: black;*/ } .o-height600 {

@@ -89,6 +89,7 @@ <script src="app.js"></script>

<script src="views/navbar.js"></script> <script src="views/main/main.js"></script> <script src="views/network/network.js"></script> + <script src="views/addressNetwork/addressNetwork.js"></script> <script src="views/sankey/sankey.js"></script> </body>
A web/views/addressNetwork/addressNetwork.html

@@ -0,0 +1,29 @@

+<div class="container"> + <div class="row"> + <div class="col-sm-3"> + <div class="panel-heading c_blueGrey300"> + <h3 class="panel-title">All addresses</h3> + </div> + <div class="panel-body" style="max-height: 500px;overflow-y: scroll;"> + <div class="form-group label-floating"> + <input ng-model="filterAddress" abmFormControl class="form-control" placeholder="Filter" type="text"> + </div> + <div ng-click="getAddressNetwork(node)" class="list-group-item" ng-repeat="node in addresses | filter: filterAddress"> + <div class="row-content"> + <p class="list-group-item-text">{{node.id}}</p> + + <!--<p class="list-group-item-text">Maecenas sed diam eget risus varius blandit.</p>--> + </div> + </div> + </div> + </div> + <div class="col-sm-9"> + <div class="panel-heading c_blueGrey300"> + <h3 class="panel-title">Network</h3> + </div> + <div class="panel-body"> + <div id="mynetwork" style="height:800px;"></div> + </div> + </div> + </div> +</div>
A web/views/addressNetwork/addressNetwork.js

@@ -0,0 +1,104 @@

+'use strict'; + +angular.module('app.addressNetwork', ['ngRoute']) + + .config(['$routeProvider', function($routeProvider) { + $routeProvider.when('/addressNetwork', { + templateUrl: 'views/addressNetwork/addressNetwork.html', + controller: 'AddressNetworkCtrl' + }); + }]) + + .controller('AddressNetworkCtrl', function($scope, $http, $routeParams) { + $scope.data = []; + $scope.addresses; + $scope.nodes = []; + $scope.edges = []; + $scope.selectedNode = {}; + var nodes, edges, container, network; + var options = { + layout: { + improvedLayout: false + }, + interaction: { + hover: true + }, + physics: { + stabilization: false, + //enabled: false + } + }; + + + $scope.showMap = function() { + var nodes = $scope.nodes; + var edges = $scope.edges; + + var container = document.getElementById('mynetwork'); + var data = { + nodes: nodes, + edges: edges + }; + network = new vis.Network(container, data, options); + network.on("click", function(params) { + params.event = "[original event]"; + //$scope.selectedNode = JSON.stringify(params, null, 4); + $scope.selectedNode = params; + console.log($scope.selectedNode); + console.log($scope.selectedNode.nodes); + var options = { + // position: {x:positionx,y:positiony}, // this is not relevant when focusing on nodes + scale: 1, + offset: { + x: 0, + y: 0 + }, + animation: { + duration: 500, + easingFunction: "easeInOutQuad" + } + }; + network.focus($scope.selectedNode.nodes[0], options); + //console.log('click event, getNodeAt returns: ' + this.getNodeAt(params.pointer.DOM)); + }); + }; + $http.get(urlapi + 'alladdresses') + .then(function(data, status, headers, config) { + console.log('data success'); + console.log(data); + $scope.addresses = data.data; + }, function(data, status, headers, config) { + console.log('data error'); + }); + + $scope.getAddressNetwork = function(address) { + console.log(address); + $http.get(urlapi + 'address/network/' + address.id) + .then(function(data, status, headers, config) { + console.log('data success'); + console.log(data); + $scope.nodes = data.data.nodes; + $scope.edges = data.data.edges; + $scope.showMap(); + }, function(data, status, headers, config) { + console.log('data error'); + }); + }; + + + $scope.focusNode = function(node) { + var options = { + // position: {x:positionx,y:positiony}, // this is not relevant when focusing on nodes + scale: 1, + offset: { + x: 0, + y: 0 + }, + animation: { + duration: 500, + easingFunction: "easeInOutQuad" + } + }; + network.focus(node.id, options); + }; + });
M web/views/navbar.html → web/views/navbar.html

@@ -1,43 +1,44 @@

<div ng-controller="NavbarCtrl"> -<div class="navbar c_grey800"> - <div class="container-fluid"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse"> + <div class="navbar c_grey800"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse"> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> - <a class="navbar-brand" href="/">goBlockchainDataAnalysis</a> - </div> - <div class="navbar-collapse collapse navbar-responsive-collapse"> - <ul class="nav navbar-nav"> - <li class="active"><a href="#!/network">Network</a></li> - <li><a href="#!/sankey">Sankey diagram</a></li> - <li><a href="javascript:void(0)">Timeline</a></li> - </ul> - <form class="navbar-form navbar-left"> - <div class="form-group"> - <input class="form-control col-md-8" placeholder="Search" type="text"> - </div> - </form> - <ul class="nav navbar-nav navbar-right"> - <li><a href="https://github.com/arnaucode/goBlockchainDataAnalysis" target="_blank">Info</a></li> - <li class="dropdown"> - <a href="bootstrap-elements.html" data-target="#" class="dropdown-toggle" data-toggle="dropdown">Settings + <a class="navbar-brand" href="/">goBlockchainDataAnalysis</a> + </div> + <div class="navbar-collapse collapse navbar-responsive-collapse"> + <ul class="nav navbar-nav"> + <li class="active"><a href="#!/network">Network</a></li> + <li><a href="#!/addressNetwork">Address Network</a></li> + <li><a href="#!/sankey">Sankey diagram</a></li> + <li><a href="javascript:void(0)">Timeline</a></li> + </ul> + <form class="navbar-form navbar-left"> + <div class="form-group"> + <input class="form-control col-md-8" placeholder="Search" type="text"> + </div> + </form> + <ul class="nav navbar-nav navbar-right"> + <li><a href="https://github.com/arnaucode/goBlockchainDataAnalysis" target="_blank">Info</a></li> + <li class="dropdown"> + <a href="bootstrap-elements.html" data-target="#" class="dropdown-toggle" data-toggle="dropdown">Settings <b class="caret"></b></a> - <ul class="dropdown-menu"> - <li><a href="javascript:void(0)">Action</a></li> - <li><a href="javascript:void(0)">Another action</a></li> - <li><a href="javascript:void(0)">Something else here</a></li> - <li class="divider"></li> - <li><a href="javascript:void(0)">Separated link</a></li> - </ul> - </li> - </ul> + <ul class="dropdown-menu"> + <li><a href="javascript:void(0)">Action</a></li> + <li><a href="javascript:void(0)">Another action</a></li> + <li><a href="javascript:void(0)">Something else here</a></li> + <li class="divider"></li> + <li><a href="javascript:void(0)">Separated link</a></li> + </ul> + </li> + </ul> + </div> </div> </div> -</div> -<!-- + <!-- <div ng-click="goBack()" class="btn"><span class="glyphicon glyphicon-arrow-left"></span> Back</div> --> </div>
M web/views/network/network.html → web/views/network/network.html

@@ -1,6 +1,26 @@

<div class="container"> <div class="row"> - <div class="col-sm-12"> + <div class="col-sm-3"> + <div class="panel-heading c_blueGrey300"> + <h3 class="panel-title">Nodes</h3> + </div> + <div class="panel-body" style="max-height: 300px;overflow-y: scroll;"> + <div ng-click="focusNode(node)" class="list-group-item" ng-repeat="node in nodes"> + <div class="row-content"> + <p class="list-group-item-text">{{node.id}}</p> + + <!--<p class="list-group-item-text">Maecenas sed diam eget risus varius blandit.</p>--> + </div> + </div> + </div> + <div class="panel-heading c_blueGrey300"> + <h3 class="panel-title">Selected node</h3> + </div> + <div class="panel-body" style="max-height: 200px;overflow-y: scroll;"> + {{selectedNode.nodes}} + </div> + </div> + <div class="col-sm-9"> <div class="panel-heading c_blueGrey300"> <h3 class="panel-title">Network</h3> </div>
M web/views/network/network.js → web/views/network/network.js

@@ -13,16 +13,19 @@ .controller('NetworkCtrl', function($scope, $http, $routeParams) {

$scope.data = []; $scope.nodes = []; $scope.edges = []; - var nodes, edges, container; + $scope.selectedNode = {}; + var nodes, edges, container, network; var options = { layout: { improvedLayout: false + }, + interaction: { + hover: true + }, + physics: { + stabilization: false, + //enabled: false } - /*, - physics:{ - //stabilization: false, - // enabled: false - }*/ };

@@ -35,7 +38,28 @@ var data = {

nodes: nodes, edges: edges }; - var network = new vis.Network(container, data, options); + network = new vis.Network(container, data, options); + network.on("click", function(params) { + params.event = "[original event]"; + //$scope.selectedNode = JSON.stringify(params, null, 4); + $scope.selectedNode = params; + console.log($scope.selectedNode); + console.log($scope.selectedNode.nodes); + var options = { + // position: {x:positionx,y:positiony}, // this is not relevant when focusing on nodes + scale: 1, + offset: { + x: 0, + y: 0 + }, + animation: { + duration: 500, + easingFunction: "easeInOutQuad" + } + }; + network.focus($scope.selectedNode.nodes[0], options); + //console.log('click event, getNodeAt returns: ' + this.getNodeAt(params.pointer.DOM)); + }); }; $http.get(urlapi + 'map')

@@ -50,4 +74,20 @@ }, function(data, status, headers, config) {

console.log('data error'); }); + + $scope.focusNode = function(node) { + var options = { + // position: {x:positionx,y:positiony}, // this is not relevant when focusing on nodes + scale: 1, + offset: { + x: 0, + y: 0 + }, + animation: { + duration: 500, + easingFunction: "easeInOutQuad" + } + }; + network.focus(node.id, options); + }; });