implemented get last blocks from blockchain since last sync and store in mongodb. Implemented generate and visualize Last7DaysHourAnalysis

README.md

@@ -33,7 +33,7 @@ rpcallowip=127.0.0.1
     "port": "3021",
 	"genesisTx": "7c27ade2c28e67ed3077f8f77b8ea6d36d4f5eba04c099be3c9faa9a4a04c046",
 	"genesisBlock": "beed44fa5e96150d95d56ebd5d2625781825a9407a5215dd7eda723373a0a1d7",
-    "startFromBlock": 160,
+    "startFromBlock": 0,
     "server": {
         "serverIP": "127.0.0.1",
         "serverPort": "3014",
@@ -68,10 +68,16 @@ Wait until the entire blockchain is downloaded.
 ./goBlockchainDataAnalysis -explore
 ```
 
-    3.1. The next runs, once the database have data, can just run:
+    3.1. The next runs, once the database have data and just need to add last blocks added in the blockchain, can just run:
+```
+./goBlockchainDataAnalysis -continue
+```
+    
+    3.2. If don't want to fill the database, can just run:
 ```
 ./goBlockchainDataAnalysis
 ```
+
 Webapp will run on 127.0.0.1:8080
 
 4. ADDITIONAL - Run the webserver, directly from the /web directory

exploreBlockchain.go

@@ -25,7 +25,6 @@ func explore(client *btcrpcclient.Client, blockHash string) {
 		check(err)
 		block, err := client.GetBlockVerbose(bh)
 		check(err)
-
 		if block.Height > config.StartFromBlock {
 			var newBlock BlockModel
 			newBlock.Hash = block.Hash
@@ -176,7 +175,7 @@ func explore(client *btcrpcclient.Client, blockHash string) {
 		blockHash = block.NextHash
 	}
 
-	fmt.Print("realBlocks (blocks with Fee and Amount values): ")
+	fmt.Print("realBlocks added (blocks with Fee and Amount values): ")
 	fmt.Println(realBlocks)
 	fmt.Println("reached the end of blockchain")
 }

main.go

@@ -4,12 +4,13 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 
 	mgo "gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
 
 	"github.com/btcsuite/btcrpcclient"
-	"github.com/fatih/color"
 	"github.com/gorilla/handlers"
 )
 
@@ -61,7 +62,7 @@ func main() {
 			for label, amount := range accounts {
 				log.Printf("%s: %s", label, amount)
 			}
-			color.Blue("starting to explore blockchain")
+			log.Println("starting to explore blockchain")
 			start := time.Now()
 			explore(client, config.GenesisBlock)
 			log.Println("blockchain exploration finished, time:")
@@ -74,6 +75,27 @@ func main() {
 			}
 			log.Printf("Block count: %d", blockCount)
 		}
+		if os.Args[1] == "-continue" {
+			// create new client instance
+			client, err := btcrpcclient.New(&btcrpcclient.ConnConfig{
+				HTTPPostMode: true,
+				DisableTLS:   true,
+				Host:         config.Host + ":" + config.Port,
+				User:         config.User,
+				Pass:         config.Pass,
+			}, nil)
+			check(err)
+			//get last block stored in mongodb
+			lastBlock := BlockModel{}
+			err = blockCollection.Find(bson.M{}).Sort("-$natural").One(&lastBlock)
+			check(err)
+			log.Println("Getting last block stored in MongoDB. Hash: " + string(lastBlock.Hash) + ", BlockHeight: " + strconv.FormatInt(lastBlock.Height, 10))
+			log.Println("continuing blockchain exploration since last block in mongodb")
+			start := time.Now()
+			explore(client, string(lastBlock.Hash))
+			log.Println("blockchain exploration finished, time:")
+			log.Println(time.Since(start))
+		}
 	}
 	//run thw webserver
 	go webserver()

mongoModels.go

@@ -92,6 +92,11 @@ type ChartAnalysisResp struct {
 	Labels []string `json:"labels"`
 	Data   []int    `json:"data"`
 }
+type ChartSeriesAnalysisResp struct {
+	Labels []string `json:"labels"`
+	Data   [][]int  `json:"data"`
+	Series []int    `json:"series"`
+}
 type DateCountModel struct {
 	Time  int64  `json:"time"`
 	Date  string `json:"date"`

serverRoutes.go

@@ -76,6 +76,12 @@ var routes = Routes{
 		"/last7day",
 		GetLast7DayAnalysis,
 	},
+	Route{
+		"GetLast7DayHourAnalysis",
+		"Get",
+		"/last7dayhour",
+		GetLast7DayHourAnalysis,
+	},
 }
 
 //ROUTES
@@ -318,3 +324,49 @@ func GetLast7DayAnalysis(w http.ResponseWriter, r *http.Request) {
 	check(err)
 	fmt.Fprintln(w, string(jsonResp))
 }
+
+func GetLast7DayHourAnalysis(w http.ResponseWriter, r *http.Request) {
+	ipFilter(w, r)
+
+	var resp ChartSeriesAnalysisResp
+
+	for i := 0; i < 7; i++ {
+		fromDate := time.Now().AddDate(0, 0, -i-1)
+		toDate := time.Now().AddDate(0, 0, -i)
+		txs := []TxModel{}
+		err := txCollection.Find(bson.M{
+			"datet": bson.M{
+				"$gt": fromDate,
+				"$lt": toDate,
+			},
+		}).Sort("-$natural").All(&txs)
+		check(err)
+
+		//generate map with 24 hours
+		hourFrequencies := map24hours()
+		for _, tx := range txs {
+			hourFrequencies[tx.Date.Hour]++
+		}
+		var hourCount []ChartCountModel
+		for hour, frequency := range hourFrequencies {
+			hourCount = append(hourCount, ChartCountModel{hour, frequency})
+		}
+		//sort by hour
+		sort.Slice(hourCount, func(i, j int) bool {
+			return hourCount[i].Elem < hourCount[j].Elem
+		})
+		var dayData []int
+		for _, d := range hourCount {
+			dayData = append(dayData, d.Count)
+		}
+		resp.Series = append(resp.Series, txs[0].Date.Day)
+		resp.Data = append(resp.Data, dayData)
+	}
+	hourLabels := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"}
+	resp.Labels = hourLabels
+
+	//convert []resp struct to json
+	jsonResp, err := json.Marshal(resp)
+	check(err)
+	fmt.Fprintln(w, string(jsonResp))
+}

web/views/dateAnalysis/dateAnalysis.html

@@ -1,11 +1,11 @@
 <div class="row">
     <div class="col-md-6">
         <div class="panel-heading c_blueGrey300">
-            <h3 class="panel-title">All time Tx/Hour</h3>
+            <h3 class="panel-title">Last 7 Days Tx/Day</h3>
         </div>
         <div class="panel-body">
-            <canvas id="line" class="chart chart-line" chart-data="totalhour.data" chart-labels="totalhour.labels">
-                  </canvas>
+            <canvas id="bar" class="chart chart-bar" chart-data="last7day.data" chart-labels="last7day.labels" chart-series="last7day.series">
+            </canvas>
         </div>
     </div>
     <div class="col-md-6">
@@ -18,3 +18,14 @@
         </div>
     </div>
 </div>
+<div class="row">
+    <div class="col-md-12">
+        <div class="panel-heading c_blueGrey300">
+            <h3 class="panel-title">Last 7 Days Tx/Hour</h3>
+        </div>
+        <div class="panel-body">
+            <canvas id="line" class="chart chart-line" chart-data="last7dayhour.data" chart-labels="last7dayhour.labels" chart-series="last7dayhour.series">
+            </canvas>
+        </div>
+    </div>
+</div>

web/views/dateAnalysis/dateAnalysis.js

@@ -9,19 +9,19 @@ angular.module('app.dateAnalysis', ['ngRoute', 'chart.js'])
         });
     }])
 
-    .controller('DateAnalysisCtrl', function($scope, $http, $routeParams) {
-        $scope.totalhour={
+    .controller('DateAnalysisCtrl', function($scope, $http) {
+        $scope.last7day={
             data: [],
             labels: []
         };
 
-        $http.get(urlapi + 'totalhouranalysis')
+        $http.get(urlapi + 'last7day')
             .then(function(data, status, headers, config) {
                 console.log('data success');
                 console.log(data);
 
-                $scope.totalhour.data = data.data.data;
-                $scope.totalhour.labels=data.data.labels;
+                $scope.last7day.data = data.data.data;
+                $scope.last7day.labels=data.data.labels;
             }, function(data, status, headers, config) {
                 console.log('data error');
             });
@@ -41,4 +41,20 @@ angular.module('app.dateAnalysis', ['ngRoute', 'chart.js'])
             }, function(data, status, headers, config) {
                 console.log('data error');
             });
+
+        $scope.last7dayhour= {
+            data:[],
+            labels:  []
+        };
+        $http.get(urlapi + 'last7dayhour')
+            .then(function(data, status, headers, config) {
+                console.log('data success');
+                console.log(data);
+
+                $scope.last7dayhour.data = data.data.data;
+                $scope.last7dayhour.labels = data.data.labels;
+                $scope.last7dayhour.series = data.data.series;
+            }, function(data, status, headers, config) {
+                console.log('data error');
+            });
     });

web/views/main/main.html

@@ -16,17 +16,19 @@
     </div>
     <div class="col-sm-4">
         <div class="panel-heading c_blueGrey300">
-            <h3 class="panel-title">Last 24 hours Tx/Hour</h3>
+            <h3 class="panel-title">Last 24 hours Tx/Hour
+            <a ng-href="#!/dateAnalysis" class="pull-right c_blueGrey300">View more</a></h3>
         </div>
         <div class="panel-body">
             <canvas id="line" class="chart chart-line" chart-data="last24hour.data" chart-labels="last24hour.labels">
             </canvas>
         </div>
         <div class="panel-heading c_blueGrey300">
-            <h3 class="panel-title">Last 7 days Tx/Day</h3>
+            <h3 class="panel-title">Last 7 days Tx/Hour
+                <a ng-href="#!/dateAnalysis" class="pull-right c_blueGrey300">View more</a></h3>
         </div>
         <div class="panel-body">
-            <canvas id="line" class="chart chart-line" chart-data="last7day.data" chart-labels="last7day.labels">
+            <canvas id="line" class="chart chart-line" chart-data="last7dayhour.data" chart-labels="last7dayhour.labels" chart-series="last7dayhour.series">
             </canvas>
         </div>
     </div>

web/views/main/main.js

@@ -49,17 +49,18 @@ angular.module('app.main', ['ngRoute'])
             }, function(data, status, headers, config) {
                 console.log('data error');
             });
-        $scope.last7day= {
+        $scope.last7dayhour= {
             data:[],
             labels:  []
         };
-        $http.get(urlapi + 'last7day')
+        $http.get(urlapi + 'last7dayhour')
             .then(function(data, status, headers, config) {
                 console.log('data success');
                 console.log(data);
 
-                $scope.last7day.data = data.data.data;
-                $scope.last7day.labels = data.data.labels;
+                $scope.last7dayhour.data = data.data.data;
+                $scope.last7dayhour.labels = data.data.labels;
+                $scope.last7dayhour.series = data.data.series;
             }, function(data, status, headers, config) {
                 console.log('data error');
             });