params: mark Geth v1.6.5 stable (Hat Trick)

core/vm: Use a bitmap instead of a map for jumpdest analysis

.mailmap

@@ -69,7 +69,7 @@ RJ Catalano <rj@erisindustries.com>
 
 Nchinda Nchinda <nchinda2@gmail.com>
 
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru> <homotopycolimit@users.noreply.github.com>
 
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 
@@ -90,3 +90,22 @@ Nick Johnson <arachnid@notdot.net>
 Henning Diedrich <hd@eonblast.com>
 Henning Diedrich <hd@eonblast.com> Drake Burroughs <wildfyre@hotmail.com>
 
+Felix Lange <fjl@twurst.com>
+Felix Lange <fjl@twurst.com> <fjl@users.noreply.github.com>
+
+Максим Чусовлянов <mchusovlianov@gmail.com>
+
+Louis Holbrook <dev@holbrook.no>
+Louis Holbrook <dev@holbrook.no> <nolash@users.noreply.github.com>
+
+Thomas Bocek <tom@tomp2p.net>
+
+Victor Tran <vu.tran54@gmail.com>
+
+Justin Drake <drakefjustin@gmail.com>
+
+Frank Wang <eternnoir@gmail.com>
+
+Gary Rong <garyrong0905@gmail.com>
+
+Guillaume Nicolas <guin56@gmail.com>

.travis.yml

@@ -5,23 +5,44 @@ matrix:
   include:
     - os: linux
       dist: trusty
-      go: 1.5.4
-      env:
-        - GO15VENDOREXPERIMENT=1
+      sudo: required
+      go: 1.7.6
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage
+
+    # These are the latest Go versions.
     - os: linux
       dist: trusty
-      go: 1.6.2
-    - os: linux
-      dist: trusty
-      go: 1.7.4
+      sudo: required
+      go: 1.8.3
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage -misspell
+
     - os: osx
-      go: 1.7.4
+      go: 1.8.3
+      sudo: required
+      script:
+        - brew update
+        - brew install caskroom/cask/brew-cask
+        - brew cask install osxfuse
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage -misspell
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.7.4
+      go: 1.8.3
       env:
         - ubuntu-ppa
         - azure-linux
@@ -53,6 +74,32 @@ matrix:
         - CC=aarch64-linux-gnu-gcc go run build/ci.go install -arch arm64
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
+    # This builder does the Linux Azure MIPS xgo uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      services:
+        - docker
+      go: 1.8.3
+      env:
+        - azure-linux-mips
+      script:
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done
+        - go run build/ci.go archive -arch mips -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mipsle --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mipsle; do mv -f "${bin}" "${bin/-linux-mipsle/}"; done
+        - go run build/ci.go archive -arch mipsle -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips64 --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips64; do mv -f "${bin}" "${bin/-linux-mips64/}"; done
+        - go run build/ci.go archive -arch mips64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips64le --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips64le; do mv -f "${bin}" "${bin/-linux-mips64le/}"; done
+        - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
     # This builder does the Android Maven and Azure uploads
     - os: linux
       dist: precise # Needed for the android tools
@@ -73,16 +120,16 @@ matrix:
         - azure-android
         - maven-android
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.8rc3.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
-        - export GOPATH=$HOME/go # Drop post Go 1.8
+        - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip -o android-ndk-r13b.zip
-        - unzip -q android-ndk-r13b.zip && rm android-ndk-r13b.zip
-        - mv android-ndk-r13b $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r13b
+        - curl https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -o android-ndk-r14b.zip
+        - unzip -q android-ndk-r14b.zip && rm android-ndk-r14b.zip
+        - mv android-ndk-r14b $HOME
+        - export ANDROID_NDK=$HOME/android-ndk-r14b
 
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum
@@ -90,7 +137,7 @@ matrix:
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: 1.7.4
+      go: 1.8.3
       env:
         - azure-osx
         - azure-ios
@@ -100,8 +147,8 @@ matrix:
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
 
         # Build the iOS framework and upload it to CocoaPods and Azure
-        - gem uninstall cocoapods -a
-        - gem install cocoapods --pre
+        - gem uninstall cocoapods -a -x
+        - gem install cocoapods
 
         - mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak
         - sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb
@@ -112,11 +159,21 @@ matrix:
 
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
+    # This builder does the Azure archive purges to avoid accumulating junk
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.8.3
+      env:
+        - azure-purge
+      script:
+        - go run build/ci.go purge -store gethstore/builds -days 14
+
 install:
   - go get golang.org/x/tools/cmd/cover
 script:
   - go run build/ci.go install
-  - go run build/ci.go test -coverage -vet -misspell
+  - go run build/ci.go test -coverage
 
 notifications:
   webhooks:

AUTHORS

@@ -3,24 +3,30 @@
 Ales Katona <ales@coinbase.com>
 Alex Leverington <alex@ethdev.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+Brian Schroeder <bts@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
 Daniel A. Nagy <nagy.da@gmail.com>
+Diego Siqueira <DiSiqueira@users.noreply.github.com>
 Elliot Shepherd <elliot@identitii.com>
 Enrique Fynn <enriquefynn@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
 Fabian Vogelsteller <fabian@frozeman.de>
 Fabio Berger <fabioberger1991@gmail.com>
 Felix Lange <fjl@twurst.com>
+Frank Wang <eternnoir@gmail.com>
+Gary Rong <garyrong0905@gmail.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guillaume Nicolas <guin56@gmail.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
 Henning Diedrich <hd@eonblast.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
 Jae Kwon <jkwon.work@gmail.com>
+Jamie Pitts <james.pitts@gmail.com>
 Jason Carver <jacarver@linkedin.com>
 Jeff R. Allen <jra@nella.org>
 Jeffrey Wilcke <jeffrey@ethereum.org>
@@ -28,15 +34,20 @@ Jens Agerberg <github@agerberg.me>
 Jonathan Brown <jbrown@bluedroplet.com>
 Joseph Chow <ethereum@outlook.com>
 Justin Clark-Casey <justincc@justincc.org>
+Justin Drake <drakefjustin@gmail.com>
 Kenji Siu <kenji@isuntv.com>
 Kobi Gurkan <kobigurk@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
+Lewis Marshall <lewis@lmars.net>
+Louis Holbrook <dev@holbrook.no>
+Luca Zeug <luclu@users.noreply.github.com>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
 Martin Holst Swende <martin@swende.se>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Micah Zoltu <micah@zoltu.net>
 Nchinda Nchinda <nchinda2@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
@@ -47,17 +58,28 @@ RJ Catalano <rj@erisindustries.com>
 Ramesh Nair <ram@hiddentao.com>
 Ricardo Catalinas Jiménez <r@untroubled.be>
 Rémy Roy <remyroy@remyroy.com>
+Shintaro Kaneko <kaneshin0120@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
 Steven Roose <stevenroose@gmail.com>
 Taylor Gerring <taylor.gerring@gmail.com>
 Thomas Bocek <tom@tomp2p.net>
 Tosh Camille <tochecamille@gmail.com>
+Valentin Wüstholz <wuestholz@users.noreply.github.com>
+Victor Farazdagi <simple.square@gmail.com>
+Victor Tran <vu.tran54@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
 Vincent G <caktux@gmail.com>
 Vitalik Buterin <v@buterin.com>
+Vivek Anand <vivekanand1101@users.noreply.github.com>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
+Zahoor Mohamed <zahoor@zahoor.in>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
+holisticode <holistic.computing@gmail.com>
+ken10100147 <sunhongping@kanjian.com>
+ligi <ligi@ligi.de>
+xiekeyang <xiekeyang@users.noreply.github.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
+Максим Чусовлянов <mchusovlianov@gmail.com>

Dockerfile

@@ -4,11 +4,12 @@ ADD . /go-ethereum
 RUN \
   apk add --update git go make gcc musl-dev linux-headers && \
   (cd go-ethereum && make geth)                           && \
-  cp go-ethereum/build/bin/geth /geth                     && \
+  cp go-ethereum/build/bin/geth /usr/local/bin/           && \
   apk del git go make gcc musl-dev linux-headers          && \
   rm -rf /go-ethereum && rm -rf /var/cache/apk/*
 
 EXPOSE 8545
 EXPOSE 30303
+EXPOSE 30303/udp
 
-ENTRYPOINT ["/geth"]
+ENTRYPOINT ["geth"]

Makefile

@@ -40,6 +40,15 @@ test: all
 clean:
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
+# The devtools target installs tools required for 'go generate'.
+# You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.
+
+devtools:
+	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
+	env GOBIN= go get -u github.com/jteeuwen/go-bindata/go-bindata
+	env GOBIN= go get -u github.com/fjl/gencodec
+	env GOBIN= go install ./cmd/abigen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios
@@ -51,12 +60,12 @@ geth-linux: geth-linux-386 geth-linux-amd64 geth-linux-arm geth-linux-mips64 get
 	@ls -ld $(GOBIN)/geth-linux-*
 
 geth-linux-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/386 -v ./cmd/geth
 	@echo "Linux 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep 386
 
 geth-linux-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/amd64 -v ./cmd/geth
 	@echo "Linux amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep amd64
 
@@ -65,32 +74,42 @@ geth-linux-arm: geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-ar
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm
 
 geth-linux-arm-5:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-5 -v ./cmd/geth
 	@echo "Linux ARMv5 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-5
 
 geth-linux-arm-6:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-6 -v ./cmd/geth
 	@echo "Linux ARMv6 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-6
 
 geth-linux-arm-7:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-7 -v ./cmd/geth
 	@echo "Linux ARMv7 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-7
 
 geth-linux-arm64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm64 -v ./cmd/geth
 	@echo "Linux ARM64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm64
 
+geth-linux-mips:
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips --ldflags '-extldflags "-static"' -v ./cmd/geth
+	@echo "Linux MIPS cross compilation done:"
+	@ls -ld $(GOBIN)/geth-linux-* | grep mips
+
+geth-linux-mipsle:
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mipsle --ldflags '-extldflags "-static"' -v ./cmd/geth
+	@echo "Linux MIPSle cross compilation done:"
+	@ls -ld $(GOBIN)/geth-linux-* | grep mipsle
+
 geth-linux-mips64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64 --ldflags '-extldflags "-static"' -v ./cmd/geth
 	@echo "Linux MIPS64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64
 
 geth-linux-mips64le:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64le --ldflags '-extldflags "-static"' -v ./cmd/geth
 	@echo "Linux MIPS64le cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64le
 
@@ -99,12 +118,12 @@ geth-darwin: geth-darwin-386 geth-darwin-amd64
 	@ls -ld $(GOBIN)/geth-darwin-*
 
 geth-darwin-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/386 -v ./cmd/geth
 	@echo "Darwin 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep 386
 
 geth-darwin-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/amd64 -v ./cmd/geth
 	@echo "Darwin amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep amd64
 
@@ -113,11 +132,11 @@ geth-windows: geth-windows-386 geth-windows-amd64
 	@ls -ld $(GOBIN)/geth-windows-*
 
 geth-windows-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/386 -v ./cmd/geth
 	@echo "Windows 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep 386
 
 geth-windows-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/amd64 -v ./cmd/geth
 	@echo "Windows amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep amd64

README.md

@@ -16,7 +16,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go and a C compiler.
+Building geth requires both a Go (version 1.7 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -70,7 +70,7 @@ This command will:
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
    as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
    This too is optional and if you leave it out you can always attach to an already running Geth instance
-   with `geth --attach`.
+   with `geth attach`.
 
 ### Full node on the Ethereum test network
 
@@ -84,21 +84,23 @@ $ geth --testnet --fast --cache=512 console
 ```
 
 The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they
-are equially useful on the testnet too. Please see above for their explanations if you've skipped to
+are equally useful on the testnet too. Please see above for their explanations if you've skipped to
 here.
 
 Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
 
  * Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
-   itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux).
+   itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
+   and Linux this also means that attaching to a running testnet node requires the use of a custom
+   endpoint since `geth attach` will try to attach to a production node endpoint by default. E.g.
+   `geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by this.
  * Instead of connecting the main Ethereum network, the client will connect to the test network,
    which uses different P2P bootnodes, different network IDs and genesis states.
-
+   
 *Note: Although there are some internal protective measures to prevent transactions from crossing
-over between the main network and test network (different starting nonces), you should make sure to
-always use separate accounts for play-money and real-money. Unless you manually move accounts, Geth
-will by default correctly separate the two networks and will not make any accounts available between
-them.*
+over between the main network and test network, you should make sure to always use separate accounts
+for play-money and real-money. Unless you manually move accounts, Geth will by default correctly
+separate the two networks and will not make any accounts available between them.*
 
 #### Docker quick start
 
@@ -118,7 +120,7 @@ As a developer, sooner rather than later you'll want to start interacting with G
 network via your own programs and not manually through the console. To aid this, Geth has built in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
-exposed via HTTP, WebSockets and IPC (unix sockets on unix based platroms, and named pipes on Windows).
+exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
 
 The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
 and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
@@ -161,6 +163,12 @@ and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`)
 
 ```json
 {
+  "config": {
+        "chainId": 0,
+        "homesteadBlock": 0,
+        "eip155Block": 0,
+        "eip158Block": 0
+    },
   "alloc"      : {},
   "coinbase"   : "0x0000000000000000000000000000000000000000",
   "difficulty" : "0x20000",

VERSION

@@ -1 +1 @@
-1.5.9
+1.6.5

accounts/abi/abi.go

@@ -17,6 +17,7 @@
 package abi
 
 import (
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -129,16 +130,15 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 	var size int
 	var offset int
 	if t.Type.IsSlice {
-
 		// get the offset which determines the start of this array ...
-		offset = int(common.BytesToBig(output[index : index+32]).Uint64())
+		offset = int(binary.BigEndian.Uint64(output[index+24 : index+32]))
 		if offset+32 > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
 		}
 
 		slice = output[offset:]
 		// ... starting with the size of the array in elements ...
-		size = int(common.BytesToBig(slice[:32]).Uint64())
+		size = int(binary.BigEndian.Uint64(slice[24:32]))
 		slice = slice[32:]
 		// ... and make sure that we've at the very least the amount of bytes
 		// available in the buffer.
@@ -147,7 +147,7 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 		}
 
 		// reslice to match the required size
-		slice = slice[:(size * 32)]
+		slice = slice[:size*32]
 	} else if t.Type.IsArray {
 		//get the number of elements in the array
 		size = t.Type.SliceSize
@@ -165,33 +165,12 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 			inter        interface{}             // interface type
 			returnOutput = slice[i*32 : i*32+32] // the return output
 		)
-
 		// set inter to the correct type (cast)
 		switch elem.T {
 		case IntTy, UintTy:
-			bigNum := common.BytesToBig(returnOutput)
-			switch t.Type.Kind {
-			case reflect.Uint8:
-				inter = uint8(bigNum.Uint64())
-			case reflect.Uint16:
-				inter = uint16(bigNum.Uint64())
-			case reflect.Uint32:
-				inter = uint32(bigNum.Uint64())
-			case reflect.Uint64:
-				inter = bigNum.Uint64()
-			case reflect.Int8:
-				inter = int8(bigNum.Int64())
-			case reflect.Int16:
-				inter = int16(bigNum.Int64())
-			case reflect.Int32:
-				inter = int32(bigNum.Int64())
-			case reflect.Int64:
-				inter = bigNum.Int64()
-			default:
-				inter = common.BytesToBig(returnOutput)
-			}
+			inter = readInteger(t.Type.Kind, returnOutput)
 		case BoolTy:
-			inter = common.BytesToBig(returnOutput).Uint64() > 0
+			inter = !allZero(returnOutput)
 		case AddressTy:
 			inter = common.BytesToAddress(returnOutput)
 		case HashTy:
@@ -207,6 +186,38 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 	return refSlice.Interface(), nil
 }
 
+func readInteger(kind reflect.Kind, b []byte) interface{} {
+	switch kind {
+	case reflect.Uint8:
+		return uint8(b[len(b)-1])
+	case reflect.Uint16:
+		return binary.BigEndian.Uint16(b[len(b)-2:])
+	case reflect.Uint32:
+		return binary.BigEndian.Uint32(b[len(b)-4:])
+	case reflect.Uint64:
+		return binary.BigEndian.Uint64(b[len(b)-8:])
+	case reflect.Int8:
+		return int8(b[len(b)-1])
+	case reflect.Int16:
+		return int16(binary.BigEndian.Uint16(b[len(b)-2:]))
+	case reflect.Int32:
+		return int32(binary.BigEndian.Uint32(b[len(b)-4:]))
+	case reflect.Int64:
+		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
+	default:
+		return new(big.Int).SetBytes(b)
+	}
+}
+
+func allZero(b []byte) bool {
+	for _, byte := range b {
+		if byte != 0 {
+			return false
+		}
+	}
+	return true
+}
+
 // toGoType parses the input and casts it to the proper type defined by the ABI
 // argument in T.
 func toGoType(i int, t Argument, output []byte) (interface{}, error) {
@@ -226,12 +237,12 @@ func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 	switch t.Type.T {
 	case StringTy, BytesTy: // variable arrays are written at the end of the return bytes
 		// parse offset from which we should start reading
-		offset := int(common.BytesToBig(output[index : index+32]).Uint64())
+		offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
 		if offset+32 > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32)
 		}
 		// parse the size up until we should be reading
-		size := int(common.BytesToBig(output[offset : offset+32]).Uint64())
+		size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
 		if offset+32+size > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size)
 		}
@@ -245,32 +256,9 @@ func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 	// convert the bytes to whatever is specified by the ABI.
 	switch t.Type.T {
 	case IntTy, UintTy:
-		bigNum := common.BytesToBig(returnOutput)
-
-		// If the type is a integer convert to the integer type
-		// specified by the ABI.
-		switch t.Type.Kind {
-		case reflect.Uint8:
-			return uint8(bigNum.Uint64()), nil
-		case reflect.Uint16:
-			return uint16(bigNum.Uint64()), nil
-		case reflect.Uint32:
-			return uint32(bigNum.Uint64()), nil
-		case reflect.Uint64:
-			return uint64(bigNum.Uint64()), nil
-		case reflect.Int8:
-			return int8(bigNum.Int64()), nil
-		case reflect.Int16:
-			return int16(bigNum.Int64()), nil
-		case reflect.Int32:
-			return int32(bigNum.Int64()), nil
-		case reflect.Int64:
-			return int64(bigNum.Int64()), nil
-		case reflect.Ptr:
-			return bigNum, nil
-		}
+		return readInteger(t.Type.Kind, returnOutput), nil
 	case BoolTy:
-		return common.BytesToBig(returnOutput).Uint64() > 0, nil
+		return !allZero(returnOutput), nil
 	case AddressTy:
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:

accounts/abi/bind/backend.go

@@ -17,13 +17,13 @@
 package bind
 
 import (
+	"context"
 	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"golang.org/x/net/context"
 )
 
 var (

accounts/abi/bind/backends/simulated.go

@@ -17,6 +17,7 @@
 package backends
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -25,6 +26,8 @@ import (
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -32,12 +35,8 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
-	"golang.org/x/net/context"
 )
 
-// Default chain configuration which sets homestead phase at block 0 (i.e. no frontier)
-var chainConfig = &params.ChainConfig{HomesteadBlock: big.NewInt(0), EIP150Block: new(big.Int), EIP158Block: new(big.Int)}
-
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
@@ -58,11 +57,12 @@ type SimulatedBackend struct {
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
-func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend {
+func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
-	core.WriteGenesisBlockForTesting(database, accounts...)
-	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux), vm.Config{})
-	backend := &SimulatedBackend{database: database, blockchain: blockchain}
+	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
+	genesis.MustCommit(database)
+	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
+	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
 	backend.rollback()
 	return backend
 }
@@ -88,7 +88,7 @@ func (b *SimulatedBackend) Rollback() {
 }
 
 func (b *SimulatedBackend) rollback() {
-	blocks, _ := core.GenerateChain(chainConfig, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {})
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {})
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), b.database)
 }
@@ -236,7 +236,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	if call.GasPrice == nil {
 		call.GasPrice = big.NewInt(1)
 	}
-	if call.Gas == nil || call.Gas.BitLen() == 0 {
+	if call.Gas == nil || call.Gas.Sign() == 0 {
 		call.Gas = big.NewInt(50000000)
 	}
 	if call.Value == nil {
@@ -244,15 +244,15 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	}
 	// Set infinite balance to the fake caller account.
 	from := statedb.GetOrNewStateObject(call.From)
-	from.SetBalance(common.MaxBig)
+	from.SetBalance(math.MaxBig256)
 	// Execute the call.
 	msg := callmsg{call}
 
-	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain)
+	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
-	vmenv := vm.NewEVM(evmContext, statedb, chainConfig, vm.Config{})
-	gaspool := new(core.GasPool).AddGas(common.MaxBig)
+	vmenv := vm.NewEVM(evmContext, statedb, b.config, vm.Config{})
+	gaspool := new(core.GasPool).AddGas(math.MaxBig256)
 	ret, gasUsed, _, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb()
 	return ret, gasUsed, err
 }
@@ -272,7 +272,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce))
 	}
 
-	blocks, _ := core.GenerateChain(chainConfig, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
 			block.AddTx(tx)
 		}

accounts/abi/bind/base.go

@@ -17,6 +17,7 @@
 package bind
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -26,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"golang.org/x/net/context"
 )
 
 // SignerFn is a signer function callback when a contract requires a method to
@@ -35,7 +35,8 @@ type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Tra
 
 // CallOpts is the collection of options to fine tune a contract call request.
 type CallOpts struct {
-	Pending bool // Whether to operate on the pending state or the last known one
+	Pending bool           // Whether to operate on the pending state or the last known one
+	From    common.Address // Optional the sender address, otherwise the first account is used
 
 	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
@@ -108,7 +109,7 @@ func (c *BoundContract) Call(opts *CallOpts, result interface{}, method string,
 		return err
 	}
 	var (
-		msg    = ethereum.CallMsg{To: &c.address, Data: input}
+		msg    = ethereum.CallMsg{From: opts.From, To: &c.address, Data: input}
 		ctx    = ensureContext(opts.Context)
 		code   []byte
 		output []byte

accounts/abi/bind/bind_test.go

@@ -169,7 +169,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy an interaction tester contract and call a transaction on it
 			_, _, interactor, err := DeployInteractor(auth, sim, "Deploy string")
@@ -210,7 +210,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, getter, err := DeployGetter(auth, sim)
@@ -242,7 +242,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, tupler, err := DeployTupler(auth, sim)
@@ -284,7 +284,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a slice tester contract and execute a n array call on it
 			_, _, slicer, err := DeploySlicer(auth, sim)
@@ -318,7 +318,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a default method invoker contract and execute its default method
 			_, _, defaulter, err := DeployDefaulter(auth, sim)
@@ -351,7 +351,7 @@ var bindTests = []struct {
 		`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
 		`
 			// Create a simulator and wrap a non-deployed contract
-			sim := backends.NewSimulatedBackend()
+			sim := backends.NewSimulatedBackend(nil)
 
 			nonexistent, err := NewNonExistent(common.Address{}, sim)
 			if err != nil {
@@ -387,7 +387,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a funky gas pattern contract
 			_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
@@ -408,6 +408,45 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Test that constant functions can be called from an (optional) specified address
+	{
+		`CallFrom`,
+		`
+			contract CallFrom {
+				function callFrom() constant returns(address) {
+					return msg.sender;
+				}
+			}
+		`, `6060604052346000575b6086806100176000396000f300606060405263ffffffff60e060020a60003504166349f8e98281146022575b6000565b34600057602c6055565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b335b905600a165627a7a72305820aef6b7685c0fa24ba6027e4870404a57df701473fe4107741805c19f5138417c0029`,
+		`[{"constant":true,"inputs":[],"name":"callFrom","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy a sender tester contract and execute a structured call on it
+			_, _, callfrom, err := DeployCallFrom(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy sender contract: %v", err)
+			}
+			sim.Commit()
+
+			if res, err := callfrom.CallFrom(nil); err != nil {
+				t.Errorf("Failed to call constant function: %v", err)
+			} else if res != (common.Address{}) {
+				t.Errorf("Invalid address returned, want: %x, got: %x", (common.Address{}), res)
+			}
+
+			for _, addr := range []common.Address{common.Address{}, common.Address{1}, common.Address{2}} {
+				if res, err := callfrom.CallFrom(&bind.CallOpts{From: addr}); err != nil {
+					t.Fatalf("Failed to call constant function: %v", err)
+				} else if res != addr {
+					t.Fatalf("Invalid address returned, want: %x, got: %x", addr, res)
+				}
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and
@@ -419,7 +458,7 @@ func TestBindings(t *testing.T) {
 		t.Skip("go sdk not found for testing")
 	}
 	// Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845)
-	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend())\n}")
+	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}")
 	linkTestDeps, err := imports.Process("", []byte(linkTestCode), nil)
 	if err != nil {
 		t.Fatalf("failed check for goimports symlink bug: %v", err)

accounts/abi/bind/util.go

@@ -17,31 +17,31 @@
 package bind
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"golang.org/x/net/context"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 // WaitMined waits for tx to be mined on the blockchain.
 // It stops waiting when the context is canceled.
 func WaitMined(ctx context.Context, b DeployBackend, tx *types.Transaction) (*types.Receipt, error) {
-	queryTicker := time.NewTicker(1 * time.Second)
+	queryTicker := time.NewTicker(time.Second)
 	defer queryTicker.Stop()
-	loghash := tx.Hash().Hex()[:8]
+
+	logger := log.New("hash", tx.Hash())
 	for {
 		receipt, err := b.TransactionReceipt(ctx, tx.Hash())
 		if receipt != nil {
 			return receipt, nil
 		}
 		if err != nil {
-			glog.V(logger.Detail).Infof("tx %x error: %v", loghash, err)
+			logger.Trace("Receipt retrieval failed", "err", err)
 		} else {
-			glog.V(logger.Detail).Infof("tx %x not yet mined...", loghash)
+			logger.Trace("Transaction not yet mined")
 		}
 		// Wait for the next round.
 		select {

accounts/abi/bind/util_test.go

@@ -17,6 +17,7 @@
 package bind_test
 
 import (
+	"context"
 	"math/big"
 	"testing"
 	"time"
@@ -27,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"golang.org/x/net/context"
 )
 
 var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
@@ -53,9 +53,8 @@ var waitDeployedTests = map[string]struct {
 
 func TestWaitDeployed(t *testing.T) {
 	for name, test := range waitDeployedTests {
-		backend := backends.NewSimulatedBackend(core.GenesisAccount{
-			Address: crypto.PubkeyToAddress(testKey.PublicKey),
-			Balance: big.NewInt(10000000000),
+		backend := backends.NewSimulatedBackend(core.GenesisAlloc{
+			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
 		})
 
 		// Create the transaction.

accounts/abi/numbers.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 var (
@@ -58,7 +59,7 @@ var (
 
 // U256 converts a big Int into a 256bit EVM number.
 func U256(n *big.Int) []byte {
-	return common.LeftPadBytes(common.U256(n).Bytes(), 32)
+	return math.PaddedBigBytes(math.U256(n), 32)
 }
 
 // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation

accounts/abi/packing.go

@@ -20,6 +20,7 @@ import (
 	"reflect"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 // packBytesSlice packs the given bytes as [L, V] as the canonical representation
@@ -45,9 +46,9 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 		return common.LeftPadBytes(reflectValue.Bytes(), 32)
 	case BoolTy:
 		if reflectValue.Bool() {
-			return common.LeftPadBytes(common.Big1.Bytes(), 32)
+			return math.PaddedBigBytes(common.Big1, 32)
 		} else {
-			return common.LeftPadBytes(common.Big0.Bytes(), 32)
+			return math.PaddedBigBytes(common.Big0, 32)
 		}
 	case BytesTy:
 		if reflectValue.Kind() == reflect.Array {

accounts/keystore/account_cache.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -30,8 +30,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -210,8 +209,8 @@ func (ac *accountCache) close() {
 // Callers must hold ac.mu.
 func (ac *accountCache) reload() {
 	accounts, err := ac.scan()
-	if err != nil && glog.V(logger.Debug) {
-		glog.Errorf("can't load keys: %v", err)
+	if err != nil {
+		log.Debug("Failed to reload keystore contents", "err", err)
 	}
 	ac.all = accounts
 	sort.Sort(ac.all)
@@ -225,7 +224,7 @@ func (ac *accountCache) reload() {
 	case ac.notify <- struct{}{}:
 	default:
 	}
-	glog.V(logger.Debug).Infof("reloaded keys, cache has %d accounts", len(ac.all))
+	log.Debug("Reloaded keystore contents", "accounts", len(ac.all))
 }
 
 func (ac *accountCache) scan() ([]accounts.Account, error) {
@@ -244,12 +243,14 @@ func (ac *accountCache) scan() ([]accounts.Account, error) {
 	for _, fi := range files {
 		path := filepath.Join(ac.keydir, fi.Name())
 		if skipKeyFile(fi) {
-			glog.V(logger.Detail).Infof("ignoring file %s", path)
+			log.Trace("Ignoring file on account scan", "path", path)
 			continue
 		}
+		logger := log.New("path", path)
+
 		fd, err := os.Open(path)
 		if err != nil {
-			glog.V(logger.Detail).Infoln(err)
+			logger.Trace("Failed to open keystore file", "err", err)
 			continue
 		}
 		buf.Reset(fd)
@@ -259,9 +260,9 @@ func (ac *accountCache) scan() ([]accounts.Account, error) {
 		addr := common.HexToAddress(keyJSON.Address)
 		switch {
 		case err != nil:
-			glog.V(logger.Debug).Infof("can't decode key %s: %v", path, err)
+			logger.Debug("Failed to decode keystore key", "err", err)
 		case (addr == common.Address{}):
-			glog.V(logger.Debug).Infof("can't decode key %s: missing or zero address", path)
+			logger.Debug("Failed to decode keystore key", "err", "missing or zero address")
 		default:
 			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
 		}

accounts/keystore/account_cache_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/key.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/crypto/secp256k1"
 	"github.com/pborman/uuid"
 )
 
@@ -125,14 +124,13 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) {
 	if err != nil {
 		return err
 	}
-
-	privkey, err := hex.DecodeString(keyJSON.PrivateKey)
+	privkey, err := crypto.HexToECDSA(keyJSON.PrivateKey)
 	if err != nil {
 		return err
 	}
 
 	k.Address = common.BytesToAddress(addr)
-	k.PrivateKey = crypto.ToECDSA(privkey)
+	k.PrivateKey = privkey
 
 	return nil
 }
@@ -157,7 +155,7 @@ func NewKeyForDirectICAP(rand io.Reader) *Key {
 		panic("key generation: could not read from random source: " + err.Error())
 	}
 	reader := bytes.NewReader(randBytes)
-	privateKeyECDSA, err := ecdsa.GenerateKey(secp256k1.S256(), reader)
+	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), reader)
 	if err != nil {
 		panic("key generation: ecdsa.GenerateKey failed: " + err.Error())
 	}
@@ -169,7 +167,7 @@ func NewKeyForDirectICAP(rand io.Reader) *Key {
 }
 
 func newKey(rand io.Reader) (*Key, error) {
-	privateKeyECDSA, err := ecdsa.GenerateKey(secp256k1.S256(), rand)
+	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand)
 	if err != nil {
 		return nil, err
 	}

accounts/keystore/keystore.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -450,7 +450,6 @@ func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (acco
 	if ks.cache.hasAddress(key.Address) {
 		return accounts.Account{}, fmt.Errorf("account already exists")
 	}
-
 	return ks.importKey(key, passphrase)
 }
 

accounts/keystore/keystore_passphrase.go

@@ -36,6 +36,7 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/randentropy"
 	"github.com/pborman/uuid"
@@ -115,8 +116,7 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		return nil, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes0 := crypto.FromECDSA(key.PrivateKey)
-	keyBytes := common.LeftPadBytes(keyBytes0, 32)
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
 	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
@@ -182,7 +182,8 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	if err != nil {
 		return nil, err
 	}
-	key := crypto.ToECDSA(keyBytes)
+	key := crypto.ToECDSAUnsafe(keyBytes)
+
 	return &Key{
 		Id:         uuid.UUID(keyId),
 		Address:    crypto.PubkeyToAddress(key.PublicKey),

accounts/keystore/keystore_passphrase_test.go

@@ -46,7 +46,7 @@ func TestKeyEncryptDecrypt(t *testing.T) {
 		// Decrypt with the correct password
 		key, err := DecryptKey(keyjson, password)
 		if err != nil {
-			t.Errorf("test %d: json key failed to decrypt: %v", i, err)
+			t.Fatalf("test %d: json key failed to decrypt: %v", i, err)
 		}
 		if key.Address != address {
 			t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address)

accounts/keystore/keystore_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/presale.go

@@ -74,7 +74,8 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 		return nil, err
 	}
 	ethPriv := crypto.Keccak256(plainText)
-	ecKey := crypto.ToECDSA(ethPriv)
+	ecKey := crypto.ToECDSAUnsafe(ethPriv)
+
 	key = &Key{
 		Id:         nil,
 		Address:    crypto.PubkeyToAddress(ecKey.PublicKey),

accounts/keystore/watch.go

@@ -21,8 +21,7 @@ package keystore
 import (
 	"time"
 
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/rjeczalik/notify"
 )
 
@@ -64,15 +63,16 @@ func (w *watcher) loop() {
 		w.starting = false
 		w.ac.mu.Unlock()
 	}()
+	logger := log.New("path", w.ac.keydir)
 
-	err := notify.Watch(w.ac.keydir, w.ev, notify.All)
-	if err != nil {
-		glog.V(logger.Detail).Infof("can't watch %s: %v", w.ac.keydir, err)
+	if err := notify.Watch(w.ac.keydir, w.ev, notify.All); err != nil {
+		logger.Trace("Failed to watch keystore folder", "err", err)
 		return
 	}
 	defer notify.Stop(w.ev)
-	glog.V(logger.Detail).Infof("now watching %s", w.ac.keydir)
-	defer glog.V(logger.Detail).Infof("no longer watching %s", w.ac.keydir)
+
+	logger.Trace("Started watching keystore folder")
+	defer logger.Trace("Stopped watching keystore folder")
 
 	w.ac.mu.Lock()
 	w.running = true

accounts/url.go

@@ -60,6 +60,15 @@ func (u URL) String() string {
 	return u.Path
 }
 
+// TerminalString implements the log.TerminalStringer interface.
+func (u URL) TerminalString() string {
+	url := u.String()
+	if len(url) > 32 {
+		return url[:31] + "…"
+	}
+	return url
+}
+
 // MarshalJSON implements the json.Marshaller interface.
 func (u URL) MarshalJSON() ([]byte, error) {
 	return json.Marshal(u.String())

accounts/usbwallet/ledger_hub.go

@@ -18,18 +18,18 @@
 // wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
 // https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
 
-// +build !ios
-
 package usbwallet
 
 import (
-	"fmt"
+	"errors"
+	"runtime"
 	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/event"
-	"github.com/karalabe/gousb/usb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/karalabe/hid"
 )
 
 // LedgerScheme is the protocol scheme prefixing account and wallet URLs.
@@ -49,8 +49,6 @@ const ledgerRefreshThrottling = 500 * time.Millisecond
 
 // LedgerHub is a accounts.Backend that can find and handle Ledger hardware wallets.
 type LedgerHub struct {
-	ctx *usb.Context // Context interfacing with a libusb instance
-
 	refreshed   time.Time               // Time instance when the list of wallets was last refreshed
 	wallets     []accounts.Wallet       // List of Ledger devices currently tracking
 	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
@@ -58,23 +56,23 @@ type LedgerHub struct {
 	updating    bool                    // Whether the event notification loop is running
 
 	quit chan chan error
-	lock sync.RWMutex
+
+	stateLock sync.RWMutex // Protects the internals of the hub from racey access
+
+	// TODO(karalabe): remove if hotplug lands on Windows
+	commsPend int        // Number of operations blocking enumeration
+	commsLock sync.Mutex // Lock protecting the pending counter and enumeration
 }
 
 // NewLedgerHub creates a new hardware wallet manager for Ledger devices.
 func NewLedgerHub() (*LedgerHub, error) {
-	// Initialize the USB library to access Ledgers through
-	ctx, err := usb.NewContext()
-	if err != nil {
-		return nil, err
+	if !hid.Supported() {
+		return nil, errors.New("unsupported platform")
 	}
-	// Create the USB hub, start and return it
 	hub := &LedgerHub{
-		ctx:  ctx,
 		quit: make(chan chan error),
 	}
 	hub.refreshWallets()
-
 	return hub, nil
 }
 
@@ -84,8 +82,8 @@ func (hub *LedgerHub) Wallets() []accounts.Wallet {
 	// Make sure the list of wallets is up to date
 	hub.refreshWallets()
 
-	hub.lock.RLock()
-	defer hub.lock.RUnlock()
+	hub.stateLock.RLock()
+	defer hub.stateLock.RUnlock()
 
 	cpy := make([]accounts.Wallet, len(hub.wallets))
 	copy(cpy, hub.wallets)
@@ -96,39 +94,49 @@ func (hub *LedgerHub) Wallets() []accounts.Wallet {
 // list of wallets based on the found devices.
 func (hub *LedgerHub) refreshWallets() {
 	// Don't scan the USB like crazy it the user fetches wallets in a loop
-	hub.lock.RLock()
+	hub.stateLock.RLock()
 	elapsed := time.Since(hub.refreshed)
-	hub.lock.RUnlock()
+	hub.stateLock.RUnlock()
 
 	if elapsed < ledgerRefreshThrottling {
 		return
 	}
 	// Retrieve the current list of Ledger devices
-	var devIDs []deviceID
-	var busIDs []uint16
+	var ledgers []hid.DeviceInfo
 
-	hub.ctx.ListDevices(func(desc *usb.Descriptor) bool {
-		// Gather Ledger devices, don't connect any just yet
+	if runtime.GOOS == "linux" {
+		// hidapi on Linux opens the device during enumeration to retrieve some infos,
+		// breaking the Ledger protocol if that is waiting for user confirmation. This
+		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
+		// need to prevent concurrent comms ourselves. The more elegant solution would
+		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// on Windows so if we need to hack it anyway, this is more elegant for now.
+		hub.commsLock.Lock()
+		if hub.commsPend > 0 { // A confirmation is pending, don't refresh
+			hub.commsLock.Unlock()
+			return
+		}
+	}
+	for _, info := range hid.Enumerate(0, 0) { // Can't enumerate directly, one valid ID is the 0 wildcard
 		for _, id := range ledgerDeviceIDs {
-			if desc.Vendor == id.Vendor && desc.Product == id.Product {
-				devIDs = append(devIDs, deviceID{Vendor: desc.Vendor, Product: desc.Product})
-				busIDs = append(busIDs, uint16(desc.Bus)<<8+uint16(desc.Address))
-				return false
+			if info.VendorID == id.Vendor && info.ProductID == id.Product {
+				ledgers = append(ledgers, info)
+				break
 			}
 		}
-		// Not ledger, ignore and don't connect either
-		return false
-	})
+	}
+	if runtime.GOOS == "linux" {
+		// See rationale before the enumeration why this is needed and only on Linux.
+		hub.commsLock.Unlock()
+	}
 	// Transform the current list of wallets into the new one
-	hub.lock.Lock()
+	hub.stateLock.Lock()
 
-	wallets := make([]accounts.Wallet, 0, len(devIDs))
+	wallets := make([]accounts.Wallet, 0, len(ledgers))
 	events := []accounts.WalletEvent{}
 
-	for i := 0; i < len(devIDs); i++ {
-		devID, busID := devIDs[i], busIDs[i]
-
-		url := accounts.URL{Scheme: LedgerScheme, Path: fmt.Sprintf("%03d:%03d", busID>>8, busID&0xff)}
+	for _, ledger := range ledgers {
+		url := accounts.URL{Scheme: LedgerScheme, Path: ledger.Path}
 
 		// Drop wallets in front of the next device or those that failed for some reason
 		for len(hub.wallets) > 0 && (hub.wallets[0].URL().Cmp(url) < 0 || hub.wallets[0].(*ledgerWallet).failed()) {
@@ -137,7 +145,7 @@ func (hub *LedgerHub) refreshWallets() {
 		}
 		// If there are no more wallets or the device is before the next, wrap new wallet
 		if len(hub.wallets) == 0 || hub.wallets[0].URL().Cmp(url) > 0 {
-			wallet := &ledgerWallet{context: hub.ctx, hardwareID: devID, locationID: busID, url: &url}
+			wallet := &ledgerWallet{hub: hub, url: &url, info: ledger, log: log.New("url", url)}
 
 			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
 			wallets = append(wallets, wallet)
@@ -156,7 +164,7 @@ func (hub *LedgerHub) refreshWallets() {
 	}
 	hub.refreshed = time.Now()
 	hub.wallets = wallets
-	hub.lock.Unlock()
+	hub.stateLock.Unlock()
 
 	// Fire all wallet events and return
 	for _, event := range events {
@@ -168,8 +176,8 @@ func (hub *LedgerHub) refreshWallets() {
 // receive notifications on the addition or removal of Ledger wallets.
 func (hub *LedgerHub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
 	// We need the mutex to reliably start/stop the update loop
-	hub.lock.Lock()
-	defer hub.lock.Unlock()
+	hub.stateLock.Lock()
+	defer hub.stateLock.Unlock()
 
 	// Subscribe the caller and track the subscriber count
 	sub := hub.updateScope.Track(hub.updateFeed.Subscribe(sink))
@@ -198,12 +206,12 @@ func (hub *LedgerHub) updater() {
 		hub.refreshWallets()
 
 		// If all our subscribers left, stop the updater
-		hub.lock.Lock()
+		hub.stateLock.Lock()
 		if hub.updateScope.Count() == 0 {
 			hub.updating = false
-			hub.lock.Unlock()
+			hub.stateLock.Unlock()
 			return
 		}
-		hub.lock.Unlock()
+		hub.stateLock.Unlock()
 	}
 }

accounts/usbwallet/ledger_wallet.go

@@ -18,11 +18,10 @@
 // wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
 // https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
 
-// +build !ios
-
 package usbwallet
 
 import (
+	"context"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
@@ -35,12 +34,11 @@ import (
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/karalabe/gousb/usb"
-	"golang.org/x/net/context"
+	"github.com/karalabe/hid"
 )
 
 // Maximum time between wallet health checks to detect USB unplugs.
@@ -74,22 +72,23 @@ const (
 	ledgerP2ReturnAddressChainCode  ledgerParam2 = 0x01 // Require a user confirmation before returning the address
 )
 
-// errReplyInvalidHeader is the error message returned by a Ledfer data exchange
+// errReplyInvalidHeader is the error message returned by a Ledger data exchange
 // if the device replies with a mismatching header. This usually means the device
 // is in browser mode.
 var errReplyInvalidHeader = errors.New("invalid reply header")
 
+// errInvalidVersionReply is the error message returned by a Ledger version retrieval
+// when a response does arrive, but it does not contain the expected data.
+var errInvalidVersionReply = errors.New("invalid version reply")
+
 // ledgerWallet represents a live USB Ledger hardware wallet.
 type ledgerWallet struct {
-	context    *usb.Context  // USB context to interface libusb through
-	hardwareID deviceID      // USB identifiers to identify this device type
-	locationID uint16        // USB bus and address to identify this device instance
-	url        *accounts.URL // Textual URL uniquely identifying this wallet
+	hub *LedgerHub    // USB hub the device originates from (TODO(karalabe): remove if hotplug lands on Windows)
+	url *accounts.URL // Textual URL uniquely identifying this wallet
 
-	device  *usb.Device  // USB device advertising itself as a Ledger wallet
-	input   usb.Endpoint // Input endpoint to send data to this device
-	output  usb.Endpoint // Output endpoint to receive data from this device
-	failure error        // Any failure that would make the device unusable
+	info    hid.DeviceInfo // Known USB device infos about the wallet
+	device  *hid.Device    // USB device advertising itself as a Ledger wallet
+	failure error          // Any failure that would make the device unusable
 
 	version  [3]byte                                    // Current version of the Ledger Ethereum app (zero if app is offline)
 	browser  bool                                       // Flag whether the Ledger is in browser mode (reply channel mismatch)
@@ -125,6 +124,8 @@ type ledgerWallet struct {
 	//     must only ever hold a *read* lock to stateLock.
 	commsLock chan struct{} // Mutex (buf=1) for the USB comms without keeping the state locked
 	stateLock sync.RWMutex  // Protects read and write access to the wallet struct fields
+
+	log log.Logger // Contextual logger to tag the ledger with its id
 }
 
 // URL implements accounts.Wallet, returning the URL of the Ledger device.
@@ -183,59 +184,12 @@ func (w *ledgerWallet) Open(passphrase string) error {
 		return accounts.ErrWalletAlreadyOpen
 	}
 	// Otherwise iterate over all USB devices and find this again (no way to directly do this)
-	// Iterate over all attached devices and fetch those seemingly Ledger
-	devices, err := w.context.ListDevices(func(desc *usb.Descriptor) bool {
-		// Only open this single specific device
-		return desc.Vendor == w.hardwareID.Vendor && desc.Product == w.hardwareID.Product &&
-			uint16(desc.Bus)<<8+uint16(desc.Address) == w.locationID
-	})
+	device, err := w.info.Open()
 	if err != nil {
 		return err
 	}
-	if len(devices) == 0 {
-		return accounts.ErrUnknownWallet
-	}
-	// Device opened, attach to the input and output endpoints
-	device := devices[0]
-
-	var invalid string
-	switch {
-	case len(device.Descriptor.Configs) == 0:
-		invalid = "no endpoint config available"
-	case len(device.Descriptor.Configs[0].Interfaces) == 0:
-		invalid = "no endpoint interface available"
-	case len(device.Descriptor.Configs[0].Interfaces[0].Setups) == 0:
-		invalid = "no endpoint setup available"
-	case len(device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints) < 2:
-		invalid = "not enough IO endpoints available"
-	}
-	if invalid != "" {
-		device.Close()
-		return fmt.Errorf("ledger wallet [%s] invalid: %s", w.url, invalid)
-	}
-	// Open the input and output endpoints to the device
-	input, err := device.OpenEndpoint(
-		device.Descriptor.Configs[0].Config,
-		device.Descriptor.Configs[0].Interfaces[0].Number,
-		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Number,
-		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints[1].Address,
-	)
-	if err != nil {
-		device.Close()
-		return fmt.Errorf("ledger wallet [%s] input open failed: %v", w.url, err)
-	}
-	output, err := device.OpenEndpoint(
-		device.Descriptor.Configs[0].Config,
-		device.Descriptor.Configs[0].Interfaces[0].Number,
-		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Number,
-		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints[0].Address,
-	)
-	if err != nil {
-		device.Close()
-		return fmt.Errorf("ledger wallet [%s] output open failed: %v", w.url, err)
-	}
 	// Wallet seems to be successfully opened, guess if the Ethereum app is running
-	w.device, w.input, w.output = device, input, output
+	w.device = device
 	w.commsLock = make(chan struct{}, 1)
 	w.commsLock <- struct{}{} // Enable lock
 
@@ -269,8 +223,8 @@ func (w *ledgerWallet) Open(passphrase string) error {
 //  - libusb on Windows doesn't support hotplug, so we can't detect USB unplugs
 //  - communication timeout on the Ledger requires a device power cycle to fix
 func (w *ledgerWallet) heartbeat() {
-	glog.V(logger.Debug).Infof("%s health-check started", w.url.String())
-	defer glog.V(logger.Debug).Infof("%s health-check stopped", w.url.String())
+	w.log.Debug("Ledger health-check started")
+	defer w.log.Debug("Ledger health-check stopped")
 
 	// Execute heartbeat checks until termination or error
 	var (
@@ -298,18 +252,18 @@ func (w *ledgerWallet) heartbeat() {
 		w.commsLock <- struct{}{}
 		w.stateLock.RUnlock()
 
-		if err == usb.ERROR_IO || err == usb.ERROR_NO_DEVICE {
+		if err != nil && err != errInvalidVersionReply {
 			w.stateLock.Lock() // Lock state to tear the wallet down
 			w.failure = err
 			w.close()
 			w.stateLock.Unlock()
 		}
-		// Ignore uninteresting errors
+		// Ignore non hardware related errors
 		err = nil
 	}
 	// In case of error, wait for termination
 	if err != nil {
-		glog.V(logger.Debug).Infof("%s health-check failed: %v", w.url.String(), err)
+		w.log.Debug("Ledger health-check failed", "err", err)
 		errc = <-w.healthQuit
 	}
 	errc <- err
@@ -363,13 +317,13 @@ func (w *ledgerWallet) close() error {
 		return nil
 	}
 	// Close the device, clear everything, then return
-	err := w.device.Close()
+	w.device.Close()
+	w.device = nil
 
-	w.device, w.input, w.output = nil, nil, nil
 	w.browser, w.version = false, [3]byte{}
 	w.accounts, w.paths = nil, nil
 
-	return err
+	return nil
 }
 
 // Accounts implements accounts.Wallet, returning the list of accounts pinned to
@@ -397,8 +351,8 @@ func (w *ledgerWallet) Accounts() []accounts.Account {
 // selfDerive is an account derivation loop that upon request attempts to find
 // new non-zero accounts.
 func (w *ledgerWallet) selfDerive() {
-	glog.V(logger.Debug).Infof("%s self-derivation started", w.url.String())
-	defer glog.V(logger.Debug).Infof("%s self-derivation stopped", w.url.String())
+	w.log.Debug("Ledger self-derivation started")
+	defer w.log.Debug("Ledger self-derivation stopped")
 
 	// Execute self-derivations until termination or error
 	var (
@@ -443,7 +397,7 @@ func (w *ledgerWallet) selfDerive() {
 			// Retrieve the next derived Ethereum account
 			if nextAddr == (common.Address{}) {
 				if nextAddr, err = w.ledgerDerive(nextPath); err != nil {
-					glog.V(logger.Warn).Infof("%s self-derivation failed: %v", w.url.String(), err)
+					w.log.Warn("Ledger account derivation failed", "err", err)
 					break
 				}
 			}
@@ -454,16 +408,16 @@ func (w *ledgerWallet) selfDerive() {
 			)
 			balance, err = w.deriveChain.BalanceAt(context, nextAddr, nil)
 			if err != nil {
-				glog.V(logger.Warn).Infof("%s self-derivation balance retrieval failed: %v", w.url.String(), err)
+				w.log.Warn("Ledger balance retrieval failed", "err", err)
 				break
 			}
 			nonce, err = w.deriveChain.NonceAt(context, nextAddr, nil)
 			if err != nil {
-				glog.V(logger.Warn).Infof("%s self-derivation nonce retrieval failed: %v", w.url.String(), err)
+				w.log.Warn("Ledger nonce retrieval failed", "err", err)
 				break
 			}
 			// If the next account is empty, stop self-derivation, but add it nonetheless
-			if balance.BitLen() == 0 && nonce == 0 {
+			if balance.Sign() == 0 && nonce == 0 {
 				empty = true
 			}
 			// We've just self-derived a new account, start tracking it locally
@@ -479,7 +433,7 @@ func (w *ledgerWallet) selfDerive() {
 
 			// Display a log message to the user for new (or previously empty accounts)
 			if _, known := w.paths[nextAddr]; !known || (!empty && nextAddr == w.deriveNextAddr) {
-				glog.V(logger.Info).Infof("%s discovered %s (balance %22v, nonce %4d) at %s", w.url.String(), nextAddr.Hex(), balance, nonce, path)
+				w.log.Info("Ledger discovered new account", "address", nextAddr, "path", path, "balance", balance, "nonce", nonce)
 			}
 			// Fetch the next potential account
 			if !empty {
@@ -518,7 +472,7 @@ func (w *ledgerWallet) selfDerive() {
 	}
 	// In case of error, wait for termination
 	if err != nil {
-		glog.V(logger.Debug).Infof("%s self-derivation failed: %s", w.url.String(), err)
+		w.log.Debug("Ledger self-derivation failed", "err", err)
 		errc = <-w.deriveQuit
 	}
 	errc <- err
@@ -623,6 +577,17 @@ func (w *ledgerWallet) SignTx(account accounts.Account, tx *types.Transaction, c
 	<-w.commsLock
 	defer func() { w.commsLock <- struct{}{} }()
 
+	// Ensure the device isn't screwed with while user confirmation is pending
+	// TODO(karalabe): remove if hotplug lands on Windows
+	w.hub.commsLock.Lock()
+	w.hub.commsPend++
+	w.hub.commsLock.Unlock()
+
+	defer func() {
+		w.hub.commsLock.Lock()
+		w.hub.commsPend--
+		w.hub.commsLock.Unlock()
+	}()
 	return w.ledgerSign(path, account.Address, tx, chainID)
 }
 
@@ -664,7 +629,7 @@ func (w *ledgerWallet) ledgerVersion() ([3]byte, error) {
 		return [3]byte{}, err
 	}
 	if len(reply) != 4 {
-		return [3]byte{}, errors.New("reply not of correct size")
+		return [3]byte{}, errInvalidVersionReply
 	}
 	// Cache the version for future reference
 	var version [3]byte
@@ -768,10 +733,6 @@ func (w *ledgerWallet) ledgerDerive(derivationPath []uint32) (common.Address, er
 //   signature R | 32 bytes
 //   signature S | 32 bytes
 func (w *ledgerWallet) ledgerSign(derivationPath []uint32, address common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	// We need to modify the timeouts to account for user feedback
-	defer func(old time.Duration) { w.device.ReadTimeout = old }(w.device.ReadTimeout)
-	w.device.ReadTimeout = time.Hour * 24 * 30 // Timeout requires a Ledger power cycle, only if you must
-
 	// Flatten the derivation path into the Ledger request
 	path := make([]byte, 1+4*len(derivationPath))
 	path[0] = byte(len(derivationPath))
@@ -902,10 +863,8 @@ func (w *ledgerWallet) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 l
 			apdu = nil
 		}
 		// Send over to the device
-		if glog.V(logger.Detail) {
-			glog.Infof("-> %03d.%03d: %x", w.device.Bus, w.device.Address, chunk)
-		}
-		if _, err := w.input.Write(chunk); err != nil {
+		w.log.Trace("Data chunk sent to the Ledger", "chunk", hexutil.Bytes(chunk))
+		if _, err := w.device.Write(chunk); err != nil {
 			return nil, err
 		}
 	}
@@ -914,12 +873,11 @@ func (w *ledgerWallet) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 l
 	chunk = chunk[:64] // Yeah, we surely have enough space
 	for {
 		// Read the next chunk from the Ledger wallet
-		if _, err := io.ReadFull(w.output, chunk); err != nil {
+		if _, err := io.ReadFull(w.device, chunk); err != nil {
 			return nil, err
 		}
-		if glog.V(logger.Detail) {
-			glog.Infof("<- %03d.%03d: %x", w.device.Bus, w.device.Address, chunk)
-		}
+		w.log.Trace("Data chunk received from the Ledger", "chunk", hexutil.Bytes(chunk))
+
 		// Make sure the transport header matches
 		if chunk[0] != 0x01 || chunk[1] != 0x01 || chunk[2] != 0x05 {
 			return nil, errReplyInvalidHeader

accounts/usbwallet/usbwallet.go

@@ -14,16 +14,12 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !ios
-
 // Package usbwallet implements support for USB hardware wallets.
 package usbwallet
 
-import "github.com/karalabe/gousb/usb"
-
 // deviceID is a combined vendor/product identifier to uniquely identify a USB
 // hardware device.
 type deviceID struct {
-	Vendor  usb.ID // The Vendor identifer
-	Product usb.ID // The Product identifier
+	Vendor  uint16 // The Vendor identifer
+	Product uint16 // The Product identifier
 }

appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-%GETH_ARCH%.zip
-  - 7z x go1.7.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.3.windows-%GETH_ARCH%.zip
+  - 7z x go1.8.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 
@@ -36,4 +36,4 @@ after_build:
 
 test_script:
   - set CGO_ENABLED=1
-  - go run build\ci.go test -vet -coverage
+  - go run build\ci.go test -coverage

build/_vendor/src/golang.org/x/net/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

build/ci-notes.md

@@ -1,18 +1,3 @@
-# Vendored Dependencies
-
-Dependencies are almost all vendored in at the standard Go `/vendor` path. This allows
-people to build go-ethereum using the standard toolchain without any particular package
-manager. It also plays nicely with `go get`, not requiring external code to be relied on.
-
-The one single dependent package missing from `vendor` is `golang.org/x/net/context`. As
-this is a package exposed via public library APIs, it must not be vendored as dependent
-code woulnd't be able to instantiate.
-
-To allow reproducible builds of go-ethereum nonetheless that don't need network access
-during build time to fetch `golang.org/x/net/context`, a version was copied into our repo
-at the very specific `/build/_vendor` path, which is added automatically by all CI build
-scripts and the makefile too.
-
 # Debian Packaging
 
 Tagged releases and develop branch commits are available as installable Debian packages
@@ -20,9 +5,9 @@ for Ubuntu. Packages are built for the all Ubuntu versions which are supported b
 Canonical:
 
 - Trusty Tahr (14.04 LTS)
-- Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
 - Yakkety Yak (16.10)
+- Zesty Zapus (17.04)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -36,18 +21,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+Add the gophers PPA and install Go 1.8 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.8 devscripts debhelper
 
 Create the source packages:
 
@@ -55,10 +40,10 @@ Create the source packages:
 
 Then go into the source package directory for your running distribution and build the package:
 
-    $ cd dist/ethereum-unstable-1.5.0+xenial
+    $ cd dist/ethereum-unstable-1.6.0+xenial
     $ dpkg-buildpackage
 
 Built packages are placed in the dist/ directory.
 
     $ cd ..
-    $ dpkg-deb -c geth-unstable_1.5.0+xenial_amd64.deb
+    $ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb

build/ci.go

@@ -23,15 +23,16 @@ Usage: go run ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [-arch architecture] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -vet ] [ -misspell ] [ packages... ]                           -- runs the tests
-   archive    [-arch architecture] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
-   importkeys                                                                                -- imports signing keys from env
-   debsrc     [ -signer key-id ] [ -upload dest ]                                            -- creates a debian source package
-   nsis                                                                                      -- creates a Windows NSIS installer
-   aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                    -- creates an Android archive
-   xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                    -- creates an iOS XCode framework
-   xgo        [ options ]                                                                    -- cross builds according to options
+   install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
+   test       [ -coverage ] [ -misspell ] [ packages... ]                                      -- runs the tests
+   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
+   importkeys                                                                                  -- imports signing keys from env
+   debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
+   nsis                                                                                        -- creates a Windows NSIS installer
+   aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
+   xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
+   xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -70,40 +71,55 @@ var (
 	allToolsArchiveFiles = []string{
 		"COPYING",
 		executablePath("abigen"),
+		executablePath("bootnode"),
 		executablePath("evm"),
 		executablePath("geth"),
-		executablePath("swarm"),
+		executablePath("puppeth"),
 		executablePath("rlpdump"),
+		executablePath("swarm"),
+		executablePath("wnode"),
 	}
 
 	// A debian package is created for all executables listed here.
 	debExecutables = []debExecutable{
 		{
-			Name:        "geth",
-			Description: "Ethereum CLI client.",
+			Name:        "abigen",
+			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
 		},
 		{
-			Name:        "rlpdump",
-			Description: "Developer utility tool that prints RLP structures.",
+			Name:        "bootnode",
+			Description: "Ethereum bootnode.",
 		},
 		{
 			Name:        "evm",
 			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
+		{
+			Name:        "geth",
+			Description: "Ethereum CLI client.",
+		},
+		{
+			Name:        "puppeth",
+			Description: "Ethereum private network manager.",
+		},
+		{
+			Name:        "rlpdump",
+			Description: "Developer utility tool that prints RLP structures.",
+		},
 		{
 			Name:        "swarm",
 			Description: "Ethereum Swarm daemon and tools",
 		},
 		{
-			Name:        "abigen",
-			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
+			Name:        "wnode",
+			Description: "Ethereum Whisper diagnostic tool",
 		},
 	}
 
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "yakkety"}
+	debDistros = []string{"trusty", "xenial", "yakkety", "zesty"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -141,6 +157,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "purge":
+		doPurge(os.Args[2:])
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -157,9 +175,9 @@ func doInstall(cmdline []string) {
 
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
-	if runtime.Version() < "go1.4" && !strings.HasPrefix(runtime.Version(), "devel") {
+	if runtime.Version() < "go1.7" && !strings.HasPrefix(runtime.Version(), "devel") {
 		log.Println("You have Go version", runtime.Version())
-		log.Println("go-ethereum requires at least Go version 1.4 and cannot")
+		log.Println("go-ethereum requires at least Go version 1.7 and cannot")
 		log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
 		os.Exit(1)
 	}
@@ -168,6 +186,8 @@ func doInstall(cmdline []string) {
 	if flag.NArg() > 0 {
 		packages = flag.Args()
 	}
+	packages = build.ExpandPackagesNoVendor(packages)
+
 	if *arch == "" || *arch == runtime.GOARCH {
 		goinstall := goTool("install", buildFlags(env)...)
 		goinstall.Args = append(goinstall.Args, "-v")
@@ -210,20 +230,16 @@ func doInstall(cmdline []string) {
 }
 
 func buildFlags(env build.Environment) (flags []string) {
-	if os.Getenv("GO_OPENCL") != "" {
-		flags = append(flags, "-tags", "opencl")
+	var ld []string
+	if env.Commit != "" {
+		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
+	}
+	if runtime.GOOS == "darwin" {
+		ld = append(ld, "-s")
 	}
 
-	// Since Go 1.5, the separator char for link time assignments
-	// is '=' and using ' ' prints a warning. However, Go < 1.5 does
-	// not support using '='.
-	sep := " "
-	if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") {
-		sep = "="
-	}
-	// Set gitCommit constant via link-time assignment.
-	if env.Commit != "" {
-		flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+env.Commit)
+	if len(ld) > 0 {
+		flags = append(flags, "-ldflags", strings.Join(ld, " "))
 	}
 	return flags
 }
@@ -244,10 +260,7 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
 		}
 	}
-	cmd.Env = []string{
-		"GO15VENDOREXPERIMENT=1",
-		"GOPATH=" + build.GOPATH(),
-	}
+	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
 	if arch == "" || arch == runtime.GOARCH {
 		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
 	} else {
@@ -269,38 +282,26 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 
 func doTest(cmdline []string) {
 	var (
-		vet      = flag.Bool("vet", false, "Whether to run go vet")
 		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
+	env := build.Env()
 
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
 		packages = flag.CommandLine.Args()
 	}
-	if len(packages) == 1 && packages[0] == "./..." {
-		// Resolve ./... manually since go vet will fail on vendored stuff
-		out, err := goTool("list", "./...").CombinedOutput()
-		if err != nil {
-			log.Fatalf("package listing failed: %v\n%s", err, string(out))
-		}
-		packages = []string{}
-		for _, line := range strings.Split(string(out), "\n") {
-			if !strings.Contains(line, "vendor") {
-				packages = append(packages, strings.TrimSpace(line))
-			}
-		}
-	}
+	packages = build.ExpandPackagesNoVendor(packages)
+
 	// Run analysis tools before the tests.
-	if *vet {
-		build.MustRun(goTool("vet", packages...))
-	}
+	build.MustRun(goTool("vet", packages...))
 	if *misspell {
-		spellcheck(packages)
+		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
+		// spellcheck(packages)
 	}
 	// Run the actual tests.
-	gotest := goTool("test")
+	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
 	gotest.Args = append(gotest.Args, "-p", "1")
@@ -440,6 +441,10 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a PR build")
 		os.Exit(0)
 	}
+	if env.IsCronJob {
+		log.Printf("skipping because this is a cron job")
+		os.Exit(0)
+	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
@@ -917,6 +922,9 @@ func newPodMetadata(env build.Environment, archive string) podMetadata {
 // Cross compilation
 
 func doXgo(cmdline []string) {
+	var (
+		alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
+	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -924,8 +932,27 @@ func doXgo(cmdline []string) {
 	gogetxgo := goTool("get", "github.com/karalabe/xgo")
 	build.MustRun(gogetxgo)
 
-	// Execute the actual cross compilation
-	xgo := xgoTool(append(buildFlags(env), flag.Args()...))
+	// If all tools building is requested, build everything the builder wants
+	args := append(buildFlags(env), flag.Args()...)
+
+	if *alltools {
+		args = append(args, []string{"--dest", GOBIN}...)
+		for _, res := range allToolsArchiveFiles {
+			if strings.HasPrefix(res, GOBIN) {
+				// Binary tool found, cross build it explicitly
+				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
+				xgo := xgoTool(args)
+				build.MustRun(xgo)
+				args = args[:len(args)-1]
+			}
+		}
+		return
+	}
+	// Otherwise xxecute the explicit cross compilation
+	path := args[len(args)-1]
+	args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
+
+	xgo := xgoTool(args)
 	build.MustRun(xgo)
 }
 
@@ -943,3 +970,62 @@ func xgoTool(args []string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// Binary distribution cleanups
+
+func doPurge(cmdline []string) {
+	var (
+		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	if env := build.Env(); !env.IsCronJob {
+		log.Printf("skipping because not a cron job")
+		os.Exit(0)
+	}
+	// Create the azure authentication and list the current archives
+	auth := build.AzureBlobstoreConfig{
+		Account:   strings.Split(*store, "/")[0],
+		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+		Container: strings.SplitN(*store, "/", 2)[1],
+	}
+	blobs, err := build.AzureBlobstoreList(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Iterate over the blobs, collect and sort all unstable builds
+	for i := 0; i < len(blobs); i++ {
+		if !strings.Contains(blobs[i].Name, "unstable") {
+			blobs = append(blobs[:i], blobs[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(blobs); i++ {
+		for j := i + 1; j < len(blobs); j++ {
+			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if iTime.After(jTime) {
+				blobs[i], blobs[j] = blobs[j], blobs[i]
+			}
+		}
+	}
+	// Filter out all archives more recent that the given threshold
+	for i, blob := range blobs {
+		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
+			blobs = blobs[:i]
+			break
+		}
+	}
+	// Delete all marked as such and return
+	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
+		log.Fatal(err)
+	}
+}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.7
+Build-Depends: debhelper (>= 8.0.0), golang-1.8
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git
@@ -13,7 +13,7 @@ Architecture: any
 Depends: ${misc:Depends}, {{.ExeList}}
 Description: Meta-package to install geth and other tools
  Meta-package to install geth and other tools
- 
+
 {{range .Executables}}
 Package: {{$.ExeName .}}
 Conflicts: {{$.ExeConflicts .}}

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/env.sh

@@ -20,8 +20,7 @@ fi
 
 # Set up the environment to use the workspace.
 GOPATH="$workspace"
-GO15VENDOREXPERIMENT=1
-export GOPATH GO15VENDOREXPERIMENT
+export GOPATH
 
 # Run the command inside the workspace.
 cd "$ethdir/go-ethereum"

build/update-license.go

@@ -47,13 +47,18 @@ var (
 		// boring stuff
 		"vendor/", "tests/files/", "build/",
 		// don't relicense vendored sources
-		"crypto/sha3/", "crypto/ecies/", "logger/glog/",
+		"cmd/internal/browser",
+		"consensus/ethash/xor.go",
+		"crypto/bn256/",
+		"crypto/ecies/",
 		"crypto/secp256k1/curve.go",
+		"crypto/sha3/",
+		"internal/jsre/deps",
+		"log/",
 		// don't license generated files
 		"contracts/chequebook/contract/",
 		"contracts/ens/contract/",
 		"contracts/release/contract.go",
-		"p2p/discv5/nodeevent_string.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
@@ -284,6 +289,9 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 		if !stat.Mode().IsRegular() {
 			continue
 		}
+		if isGenerated(file) {
+			continue
+		}
 		info, err := fileInfo(file)
 		if err != nil {
 			fmt.Printf("ERROR %s: %v\n", file, err)
@@ -294,6 +302,23 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
+func isGenerated(file string) bool {
+	fd, err := os.Open(file)
+	if err != nil {
+		return false
+	}
+	defer fd.Close()
+	buf := make([]byte, 2048)
+	n, _ := fd.Read(buf)
+	buf = buf[:n]
+	for _, l := range bytes.Split(buf, []byte("\n")) {
+		if bytes.HasPrefix(l, []byte("// Code generated")) {
+			return true
+		}
+	}
+	return false
+}
+
 // fileInfo finds the lowest year in which the given file was committed.
 func fileInfo(file string) (*info, error) {
 	info := &info{file: file, Year: int64(time.Now().Year())}

cmd/bootnode/main.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
 	"github.com/ethereum/go-ethereum/p2p/nat"
@@ -42,15 +42,19 @@ func main() {
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
 		netrestrict = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)")
 		runv5       = flag.Bool("v5", false, "run a v5 topic discovery bootnode")
+		verbosity   = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-9)")
+		vmodule     = flag.String("vmodule", "", "log verbosity pattern")
 
 		nodeKey *ecdsa.PrivateKey
 		err     error
 	)
-	flag.Var(glog.GetVerbosity(), "verbosity", "log verbosity (0-9)")
-	flag.Var(glog.GetVModule(), "vmodule", "log verbosity pattern")
-	glog.SetToStderr(true)
 	flag.Parse()
 
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
+	glogger.Verbosity(log.Lvl(*verbosity))
+	glogger.Vmodule(*vmodule)
+	log.Root().SetHandler(glogger)
+
 	natm, err := nat.Parse(*natdesc)
 	if err != nil {
 		utils.Fatalf("-nat: %v", err)
@@ -64,6 +68,7 @@ func main() {
 		if err = crypto.SaveECDSA(*genKey, nodeKey); err != nil {
 			utils.Fatalf("%v", err)
 		}
+		return
 	case *nodeKeyFile == "" && *nodeKeyHex == "":
 		utils.Fatalf("Use -nodekey or -nodekeyhex to specify a private key")
 	case *nodeKeyFile != "" && *nodeKeyHex != "":

cmd/disasm/main.go

@@ -1,60 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// disasm is a pretty-printer for EVM bytecode.
-package main
-
-import (
-	"encoding/hex"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/core/vm"
-)
-
-func main() {
-	code, err := ioutil.ReadAll(os.Stdin)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	code, err = hex.DecodeString(strings.TrimSpace(string(code[:])))
-	if err != nil {
-		fmt.Printf("Error: %v\n", err)
-		return
-	}
-	fmt.Printf("%x\n", code)
-
-	for pc := uint64(0); pc < uint64(len(code)); pc++ {
-		op := vm.OpCode(code[pc])
-
-		switch op {
-		case vm.PUSH1, vm.PUSH2, vm.PUSH3, vm.PUSH4, vm.PUSH5, vm.PUSH6, vm.PUSH7, vm.PUSH8, vm.PUSH9, vm.PUSH10, vm.PUSH11, vm.PUSH12, vm.PUSH13, vm.PUSH14, vm.PUSH15, vm.PUSH16, vm.PUSH17, vm.PUSH18, vm.PUSH19, vm.PUSH20, vm.PUSH21, vm.PUSH22, vm.PUSH23, vm.PUSH24, vm.PUSH25, vm.PUSH26, vm.PUSH27, vm.PUSH28, vm.PUSH29, vm.PUSH30, vm.PUSH31, vm.PUSH32:
-			a := uint64(op) - uint64(vm.PUSH1) + 1
-			u := pc + 1 + a
-			if uint64(len(code)) <= pc || uint64(len(code)) < u {
-				fmt.Printf("Error: incomplete push instruction at %v\n", pc)
-				return
-			}
-			fmt.Printf("%-5d  %v  => %x\n", pc, op, code[pc+1:u])
-			pc += a
-		default:
-			fmt.Printf("%-5d  %v\n", pc, op)
-		}
-	}
-}

cmd/ethtest/main.go

@@ -1,222 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// ethtest executes Ethereum JSON tests.
-package main
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/tests"
-	"gopkg.in/urfave/cli.v1"
-)
-
-var (
-	continueOnError = false
-	testExtension   = ".json"
-	defaultTest     = "all"
-	defaultDir      = "."
-	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"}
-	testDirMapping  = map[string]string{"BlockTests": "BlockchainTests"}
-	skipTests       = []string{}
-
-	TestFlag = cli.StringFlag{
-		Name:  "test",
-		Usage: "Test type (string): VMTests, TransactionTests, StateTests, BlockTests",
-		Value: defaultTest,
-	}
-	FileFlag = cli.StringFlag{
-		Name:   "file",
-		Usage:  "Test file or directory. Directories are searched for .json files 1 level deep",
-		Value:  defaultDir,
-		EnvVar: "ETHEREUM_TEST_PATH",
-	}
-	ContinueOnErrorFlag = cli.BoolFlag{
-		Name:  "continue",
-		Usage: "Continue running tests on error (true) or [default] exit immediately (false)",
-	}
-	ReadStdInFlag = cli.BoolFlag{
-		Name:  "stdin",
-		Usage: "Accept input from stdin instead of reading from file",
-	}
-	SkipTestsFlag = cli.StringFlag{
-		Name:  "skip",
-		Usage: "Tests names to skip",
-	}
-	TraceFlag = cli.BoolFlag{
-		Name:  "trace",
-		Usage: "Enable VM tracing",
-	}
-)
-
-func runTestWithReader(test string, r io.Reader) error {
-	glog.Infoln("runTest", test)
-	var err error
-	switch strings.ToLower(test) {
-	case "bk", "block", "blocktest", "blockchaintest", "blocktests", "blockchaintests":
-		err = tests.RunBlockTestWithReader(params.MainNetHomesteadBlock, params.MainNetDAOForkBlock, params.MainNetHomesteadGasRepriceBlock, r, skipTests)
-	case "st", "state", "statetest", "statetests":
-		rs := &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock}
-		err = tests.RunStateTestWithReader(rs, r, skipTests)
-	case "tx", "transactiontest", "transactiontests":
-		rs := &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock}
-		err = tests.RunTransactionTestsWithReader(rs, r, skipTests)
-	case "vm", "vmtest", "vmtests":
-		err = tests.RunVmTestWithReader(r, skipTests)
-	case "rlp", "rlptest", "rlptests":
-		err = tests.RunRLPTestWithReader(r, skipTests)
-	default:
-		err = fmt.Errorf("Invalid test type specified: %v", test)
-	}
-	return err
-}
-
-func getFiles(path string) ([]string, error) {
-	glog.Infoln("getFiles", path)
-	var files []string
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	switch mode := fi.Mode(); {
-	case mode.IsDir():
-		fi, _ := ioutil.ReadDir(path)
-		files = make([]string, len(fi))
-		for i, v := range fi {
-			// only go 1 depth and leave directory entires blank
-			if !v.IsDir() && v.Name()[len(v.Name())-len(testExtension):len(v.Name())] == testExtension {
-				files[i] = filepath.Join(path, v.Name())
-				glog.Infoln("Found file", files[i])
-			}
-		}
-	case mode.IsRegular():
-		files = make([]string, 1)
-		files[0] = path
-	}
-
-	return files, nil
-}
-
-func runSuite(test, file string) {
-	var tests []string
-
-	if test == defaultTest {
-		tests = allTests
-	} else {
-		tests = []string{test}
-	}
-
-	for _, curTest := range tests {
-		glog.Infoln("runSuite", curTest, file)
-		var err error
-		var files []string
-		if test == defaultTest {
-			// check if we have an explicit directory mapping for the test
-			if _, ok := testDirMapping[curTest]; ok {
-				files, err = getFiles(filepath.Join(file, testDirMapping[curTest]))
-			} else {
-				// otherwise assume test name
-				files, err = getFiles(filepath.Join(file, curTest))
-			}
-		} else {
-			files, err = getFiles(file)
-		}
-		if err != nil {
-			glog.Fatalln(err)
-		}
-
-		if len(files) == 0 {
-			glog.Warningln("No files matched path")
-		}
-		for _, curFile := range files {
-			// Skip blank entries
-			if len(curFile) == 0 {
-				continue
-			}
-
-			r, err := os.Open(curFile)
-			if err != nil {
-				glog.Fatalln(err)
-			}
-			defer r.Close()
-
-			err = runTestWithReader(curTest, r)
-			if err != nil {
-				if continueOnError {
-					glog.Errorln(err)
-				} else {
-					glog.Fatalln(err)
-				}
-			}
-		}
-	}
-}
-
-func setupApp(c *cli.Context) error {
-	flagTest := c.GlobalString(TestFlag.Name)
-	flagFile := c.GlobalString(FileFlag.Name)
-	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
-	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
-	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")
-
-	if !useStdIn {
-		runSuite(flagTest, flagFile)
-	} else {
-		if err := runTestWithReader(flagTest, os.Stdin); err != nil {
-			glog.Fatalln(err)
-		}
-	}
-	return nil
-}
-
-func main() {
-	glog.SetToStderr(true)
-
-	app := cli.NewApp()
-	app.Name = "ethtest"
-	app.Usage = "go-ethereum test interface"
-	app.Action = setupApp
-	app.Version = "0.2.0"
-	app.Author = "go-ethereum team"
-
-	app.Flags = []cli.Flag{
-		TestFlag,
-		FileFlag,
-		ContinueOnErrorFlag,
-		ReadStdInFlag,
-		SkipTestsFlag,
-		TraceFlag,
-	}
-
-	if err := app.Run(os.Args); err != nil {
-		glog.Fatalln(err)
-	}
-
-}

cmd/evm/compiler.go

@@ -0,0 +1,55 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var compileCommand = cli.Command{
+	Action:    compileCmd,
+	Name:      "compile",
+	Usage:     "compiles easm source to evm binary",
+	ArgsUsage: "<file>",
+}
+
+func compileCmd(ctx *cli.Context) error {
+	debug := ctx.GlobalBool(DebugFlag.Name)
+
+	if len(ctx.Args().First()) == 0 {
+		return errors.New("filename required")
+	}
+
+	fn := ctx.Args().First()
+	src, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return err
+	}
+
+	bin, err := compiler.Compile(fn, src, debug)
+	if err != nil {
+		return err
+	}
+	fmt.Println(bin)
+	return nil
+}

cmd/evm/disasm.go

@@ -0,0 +1,53 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/core/asm"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var disasmCommand = cli.Command{
+	Action:    disasmCmd,
+	Name:      "disasm",
+	Usage:     "disassembles evm binary",
+	ArgsUsage: "<file>",
+}
+
+func disasmCmd(ctx *cli.Context) error {
+	if len(ctx.Args().First()) == 0 {
+		return errors.New("filename required")
+	}
+
+	fn := ctx.Args().First()
+	in, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return err
+	}
+
+	code := strings.TrimSpace(string(in[:]))
+	fmt.Printf("%v\n", code)
+	if err = asm.PrintDisassembled(code); err != nil {
+		return err
+	}
+	return nil
+}

cmd/evm/internal/compiler/compiler.go

@@ -0,0 +1,39 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/asm"
+)
+
+func Compile(fn string, src []byte, debug bool) (string, error) {
+	compiler := asm.NewCompiler(debug)
+	compiler.Feed(asm.Lex(fn, src, debug))
+
+	bin, compileErrors := compiler.Compile()
+	if len(compileErrors) > 0 {
+		// report errors
+		for _, err := range compileErrors {
+			fmt.Printf("%s:%v\n", fn, err)
+		}
+		return "", errors.New("compiling failed")
+	}
+	return bin, nil
+}

cmd/evm/main.go

@@ -19,19 +19,10 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"math/big"
 	"os"
-	goruntime "runtime"
-	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/core/vm/runtime"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger/glog"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -44,6 +35,18 @@ var (
 		Name:  "debug",
 		Usage: "output full trace logs",
 	}
+	MemProfileFlag = cli.StringFlag{
+		Name:  "memprofile",
+		Usage: "creates a memory profile at the given path",
+	}
+	CPUProfileFlag = cli.StringFlag{
+		Name:  "cpuprofile",
+		Usage: "creates a CPU profile at the given path",
+	}
+	StatDumpFlag = cli.BoolFlag{
+		Name:  "statdump",
+		Usage: "displays stack and heap memory information",
+	}
 	CodeFlag = cli.StringFlag{
 		Name:  "code",
 		Usage: "EVM code",
@@ -52,20 +55,20 @@ var (
 		Name:  "codefile",
 		Usage: "file containing EVM code",
 	}
-	GasFlag = cli.StringFlag{
+	GasFlag = cli.Uint64Flag{
 		Name:  "gas",
 		Usage: "gas limit for the evm",
-		Value: "10000000000",
+		Value: 10000000000,
 	}
-	PriceFlag = cli.StringFlag{
+	PriceFlag = utils.BigFlag{
 		Name:  "price",
 		Usage: "price set for the evm",
-		Value: "0",
+		Value: new(big.Int),
 	}
-	ValueFlag = cli.StringFlag{
+	ValueFlag = utils.BigFlag{
 		Name:  "value",
 		Usage: "value set for the evm",
-		Value: "0",
+		Value: new(big.Int),
 	}
 	DumpFlag = cli.BoolFlag{
 		Name:  "dump",
@@ -75,10 +78,6 @@ var (
 		Name:  "input",
 		Usage: "input for the EVM",
 	}
-	SysStatFlag = cli.BoolFlag{
-		Name:  "sysstat",
-		Usage: "display system stats",
-	}
 	VerbosityFlag = cli.IntFlag{
 		Name:  "verbosity",
 		Usage: "sets the verbosity level",
@@ -98,7 +97,6 @@ func init() {
 		CreateFlag,
 		DebugFlag,
 		VerbosityFlag,
-		SysStatFlag,
 		CodeFlag,
 		CodeFileFlag,
 		GasFlag,
@@ -107,108 +105,15 @@ func init() {
 		DumpFlag,
 		InputFlag,
 		DisableGasMeteringFlag,
+		MemProfileFlag,
+		CPUProfileFlag,
+		StatDumpFlag,
 	}
-	app.Action = run
-}
-
-func run(ctx *cli.Context) error {
-	glog.SetToStderr(true)
-	glog.SetV(ctx.GlobalInt(VerbosityFlag.Name))
-
-	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, db)
-	sender := statedb.CreateAccount(common.StringToAddress("sender"))
-
-	logger := vm.NewStructLogger(nil)
-
-	tstart := time.Now()
-
-	var (
-		code []byte
-		ret  []byte
-		err  error
-	)
-
-	if ctx.GlobalString(CodeFlag.Name) != "" {
-		code = common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
-	} else {
-		var hexcode []byte
-		if ctx.GlobalString(CodeFileFlag.Name) != "" {
-			var err error
-			hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name))
-			if err != nil {
-				fmt.Printf("Could not load code from file: %v\n", err)
-				os.Exit(1)
-			}
-		} else {
-			var err error
-			hexcode, err = ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				fmt.Printf("Could not load code from stdin: %v\n", err)
-				os.Exit(1)
-			}
-		}
-		code = common.Hex2Bytes(string(hexcode[:]))
+	app.Commands = []cli.Command{
+		compileCommand,
+		disasmCommand,
+		runCommand,
 	}
-
-	if ctx.GlobalBool(CreateFlag.Name) {
-		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
-		ret, _, err = runtime.Create(input, &runtime.Config{
-			Origin:   sender.Address(),
-			State:    statedb,
-			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
-			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
-			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
-			EVMConfig: vm.Config{
-				Tracer:             logger,
-				Debug:              ctx.GlobalBool(DebugFlag.Name),
-				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
-			},
-		})
-	} else {
-		receiver := statedb.CreateAccount(common.StringToAddress("receiver"))
-		receiver.SetCode(crypto.Keccak256Hash(code), code)
-
-		ret, err = runtime.Call(receiver.Address(), common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtime.Config{
-			Origin:   sender.Address(),
-			State:    statedb,
-			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
-			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
-			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
-			EVMConfig: vm.Config{
-				Tracer:             logger,
-				Debug:              ctx.GlobalBool(DebugFlag.Name),
-				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
-			},
-		})
-	}
-	vmdone := time.Since(tstart)
-
-	if ctx.GlobalBool(DumpFlag.Name) {
-		statedb.Commit(true)
-		fmt.Println(string(statedb.Dump()))
-	}
-	vm.StdErrFormat(logger.StructLogs())
-
-	if ctx.GlobalBool(SysStatFlag.Name) {
-		var mem goruntime.MemStats
-		goruntime.ReadMemStats(&mem)
-		fmt.Printf("vm took %v\n", vmdone)
-		fmt.Printf(`alloc:      %d
-tot alloc:  %d
-no. malloc: %d
-heap alloc: %d
-heap objs:  %d
-num gc:     %d
-`, mem.Alloc, mem.TotalAlloc, mem.Mallocs, mem.HeapAlloc, mem.HeapObjects, mem.NumGC)
-	}
-
-	fmt.Printf("OUT: 0x%x", ret)
-	if err != nil {
-		fmt.Printf(" error: %v", err)
-	}
-	fmt.Println()
-	return nil
 }
 
 func main() {

cmd/evm/runner.go

@@ -0,0 +1,180 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime/pprof"
+	"time"
+
+	goruntime "runtime"
+
+	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/core/vm/runtime"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var runCommand = cli.Command{
+	Action:      runCmd,
+	Name:        "run",
+	Usage:       "run arbitrary evm binary",
+	ArgsUsage:   "<code>",
+	Description: `The run command runs arbitrary EVM code.`,
+}
+
+func runCmd(ctx *cli.Context) error {
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
+	glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name)))
+	log.Root().SetHandler(glogger)
+
+	var (
+		db, _      = ethdb.NewMemDatabase()
+		statedb, _ = state.New(common.Hash{}, db)
+		sender     = common.StringToAddress("sender")
+		logger     = vm.NewStructLogger(nil)
+	)
+	statedb.CreateAccount(sender)
+
+	var (
+		code []byte
+		ret  []byte
+		err  error
+	)
+	if fn := ctx.Args().First(); len(fn) > 0 {
+		src, err := ioutil.ReadFile(fn)
+		if err != nil {
+			return err
+		}
+
+		bin, err := compiler.Compile(fn, src, false)
+		if err != nil {
+			return err
+		}
+		code = common.Hex2Bytes(bin)
+	} else if ctx.GlobalString(CodeFlag.Name) != "" {
+		code = common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
+	} else {
+		var hexcode []byte
+		if ctx.GlobalString(CodeFileFlag.Name) != "" {
+			var err error
+			hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name))
+			if err != nil {
+				fmt.Printf("Could not load code from file: %v\n", err)
+				os.Exit(1)
+			}
+		} else {
+			var err error
+			hexcode, err = ioutil.ReadAll(os.Stdin)
+			if err != nil {
+				fmt.Printf("Could not load code from stdin: %v\n", err)
+				os.Exit(1)
+			}
+		}
+		code = common.Hex2Bytes(string(bytes.TrimRight(hexcode, "\n")))
+	}
+
+	runtimeConfig := runtime.Config{
+		Origin:   sender,
+		State:    statedb,
+		GasLimit: ctx.GlobalUint64(GasFlag.Name),
+		GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
+		Value:    utils.GlobalBig(ctx, ValueFlag.Name),
+		EVMConfig: vm.Config{
+			Tracer:             logger,
+			Debug:              ctx.GlobalBool(DebugFlag.Name),
+			DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
+		},
+	}
+
+	if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" {
+		f, err := os.Create(cpuProfilePath)
+		if err != nil {
+			fmt.Println("could not create CPU profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			fmt.Println("could not start CPU profile: ", err)
+			os.Exit(1)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	tstart := time.Now()
+	if ctx.GlobalBool(CreateFlag.Name) {
+		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
+		ret, _, err = runtime.Create(input, &runtimeConfig)
+	} else {
+		receiver := common.StringToAddress("receiver")
+		statedb.SetCode(receiver, code)
+
+		ret, err = runtime.Call(receiver, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtimeConfig)
+	}
+	execTime := time.Since(tstart)
+
+	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
+		fmt.Println(string(statedb.Dump()))
+	}
+
+	if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {
+		f, err := os.Create(memProfilePath)
+		if err != nil {
+			fmt.Println("could not create memory profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			fmt.Println("could not write memory profile: ", err)
+			os.Exit(1)
+		}
+		f.Close()
+	}
+
+	if ctx.GlobalBool(DebugFlag.Name) {
+		fmt.Fprintln(os.Stderr, "#### TRACE ####")
+		vm.WriteTrace(os.Stderr, logger.StructLogs())
+		fmt.Fprintln(os.Stderr, "#### LOGS ####")
+		vm.WriteLogs(os.Stderr, statedb.Logs())
+	}
+
+	if ctx.GlobalBool(StatDumpFlag.Name) {
+		var mem goruntime.MemStats
+		goruntime.ReadMemStats(&mem)
+		fmt.Fprintf(os.Stderr, `evm execution time: %v
+heap objects:       %d
+allocations:        %d
+total allocations:  %d
+GC calls:           %d
+
+`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC)
+	}
+
+	fmt.Printf("0x%x", ret)
+	if err != nil {
+		fmt.Printf(" error: %v", err)
+	}
+	fmt.Println()
+	return nil
+}

cmd/faucet/faucet.go

@@ -0,0 +1,543 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// faucet is a Ether faucet backed by a light client.
+package main
+
+//go:generate go-bindata -nometadata -o website.go faucet.html
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"math"
+	"math/big"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/eth/downloader"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/ethstats"
+	"github.com/ethereum/go-ethereum/les"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/nat"
+	"github.com/ethereum/go-ethereum/params"
+	"golang.org/x/net/websocket"
+)
+
+var (
+	genesisFlag = flag.String("genesis", "", "Genesis json file to seed the chain with")
+	apiPortFlag = flag.Int("apiport", 8080, "Listener port for the HTTP API connection")
+	ethPortFlag = flag.Int("ethport", 30303, "Listener port for the devp2p connection")
+	bootFlag    = flag.String("bootnodes", "", "Comma separated bootnode enode URLs to seed with")
+	netFlag     = flag.Uint64("network", 0, "Network ID to use for the Ethereum protocol")
+	statsFlag   = flag.String("ethstats", "", "Ethstats network monitoring auth string")
+
+	netnameFlag = flag.String("faucet.name", "", "Network name to assign to the faucet")
+	payoutFlag  = flag.Int("faucet.amount", 1, "Number of Ethers to pay out per user request")
+	minutesFlag = flag.Int("faucet.minutes", 1440, "Number of minutes to wait between funding rounds")
+	tiersFlag   = flag.Int("faucet.tiers", 3, "Number of funding tiers to enable (x3 time, x2.5 funds)")
+
+	accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with")
+	accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds")
+
+	githubUser  = flag.String("github.user", "", "GitHub user to authenticate with for Gist access")
+	githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with")
+
+	captchaToken  = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side")
+	captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side")
+
+	logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+)
+
+var (
+	ether = new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
+)
+
+func main() {
+	// Parse the flags and set up the logger to print everything requested
+	flag.Parse()
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*logFlag), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+
+	// Construct the payout tiers
+	amounts := make([]string, *tiersFlag)
+	periods := make([]string, *tiersFlag)
+	for i := 0; i < *tiersFlag; i++ {
+		// Calculate the amount for the next tier and format it
+		amount := float64(*payoutFlag) * math.Pow(2.5, float64(i))
+		amounts[i] = fmt.Sprintf("%s Ethers", strconv.FormatFloat(amount, 'f', -1, 64))
+		if amount == 1 {
+			amounts[i] = strings.TrimSuffix(amounts[i], "s")
+		}
+		// Calculate the period for the next tier and format it
+		period := *minutesFlag * int(math.Pow(3, float64(i)))
+		periods[i] = fmt.Sprintf("%d mins", period)
+		if period%60 == 0 {
+			period /= 60
+			periods[i] = fmt.Sprintf("%d hours", period)
+
+			if period%24 == 0 {
+				period /= 24
+				periods[i] = fmt.Sprintf("%d days", period)
+			}
+		}
+		if period == 1 {
+			periods[i] = strings.TrimSuffix(periods[i], "s")
+		}
+	}
+	// Load up and render the faucet website
+	tmpl, err := Asset("faucet.html")
+	if err != nil {
+		log.Crit("Failed to load the faucet template", "err", err)
+	}
+	website := new(bytes.Buffer)
+	err = template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{
+		"Network":   *netnameFlag,
+		"Amounts":   amounts,
+		"Periods":   periods,
+		"Recaptcha": *captchaToken,
+	})
+	if err != nil {
+		log.Crit("Failed to render the faucet template", "err", err)
+	}
+	// Load and parse the genesis block requested by the user
+	blob, err := ioutil.ReadFile(*genesisFlag)
+	if err != nil {
+		log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err)
+	}
+	genesis := new(core.Genesis)
+	if err = json.Unmarshal(blob, genesis); err != nil {
+		log.Crit("Failed to parse genesis block json", "err", err)
+	}
+	// Convert the bootnodes to internal enode representations
+	var enodes []*discv5.Node
+	for _, boot := range strings.Split(*bootFlag, ",") {
+		if url, err := discv5.ParseNode(boot); err == nil {
+			enodes = append(enodes, url)
+		} else {
+			log.Error("Failed to parse bootnode URL", "url", boot, "err", err)
+		}
+	}
+	// Load up the account key and decrypt its password
+	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
+		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
+	}
+	pass := string(blob)
+
+	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
+	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
+		log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err)
+	}
+	acc, err := ks.Import(blob, pass, pass)
+	if err != nil {
+		log.Crit("Failed to import faucet signer account", "err", err)
+	}
+	ks.Unlock(acc, pass)
+
+	// Assemble and start the faucet light service
+	faucet, err := newFaucet(genesis, *ethPortFlag, enodes, *netFlag, *statsFlag, ks, website.Bytes())
+	if err != nil {
+		log.Crit("Failed to start faucet", "err", err)
+	}
+	defer faucet.close()
+
+	if err := faucet.listenAndServe(*apiPortFlag); err != nil {
+		log.Crit("Failed to launch faucet API", "err", err)
+	}
+}
+
+// request represents an accepted funding request.
+type request struct {
+	Username string             `json:"username"` // GitHub user for displaying an avatar
+	Account  common.Address     `json:"account"`  // Ethereum address being funded
+	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
+	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+}
+
+// faucet represents a crypto faucet backed by an Ethereum light client.
+type faucet struct {
+	config *params.ChainConfig // Chain configurations for signing
+	stack  *node.Node          // Ethereum protocol stack
+	client *ethclient.Client   // Client connection to the Ethereum chain
+	index  []byte              // Index page to serve up on the web
+
+	keystore *keystore.KeyStore // Keystore containing the single signer
+	account  accounts.Account   // Account funding user faucet requests
+	nonce    uint64             // Current pending nonce of the faucet
+	price    *big.Int           // Current gas price to issue funds with
+
+	conns    []*websocket.Conn    // Currently live websocket connections
+	timeouts map[string]time.Time // History of users and their funding timeouts
+	reqs     []*request           // Currently pending funding requests
+	update   chan struct{}        // Channel to signal request updates
+
+	lock sync.RWMutex // Lock protecting the faucet's internals
+}
+
+func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
+	// Assemble the raw devp2p protocol stack
+	stack, err := node.New(&node.Config{
+		Name:    "geth",
+		Version: params.Version,
+		DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
+		P2P: p2p.Config{
+			NAT:              nat.Any(),
+			NoDiscovery:      true,
+			DiscoveryV5:      true,
+			ListenAddr:       fmt.Sprintf(":%d", port),
+			DiscoveryV5Addr:  fmt.Sprintf(":%d", port+1),
+			MaxPeers:         25,
+			BootstrapNodesV5: enodes,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// Assemble the Ethereum light client protocol
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		cfg := eth.DefaultConfig
+		cfg.SyncMode = downloader.LightSync
+		cfg.NetworkId = network
+		cfg.Genesis = genesis
+		return les.New(ctx, &cfg)
+	}); err != nil {
+		return nil, err
+	}
+	// Assemble the ethstats monitoring and reporting service'
+	if stats != "" {
+		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			var serv *les.LightEthereum
+			ctx.Service(&serv)
+			return ethstats.New(stats, nil, serv)
+		}); err != nil {
+			return nil, err
+		}
+	}
+	// Boot up the client and ensure it connects to bootnodes
+	if err := stack.Start(); err != nil {
+		return nil, err
+	}
+	for _, boot := range enodes {
+		old, _ := discover.ParseNode(boot.String())
+		stack.Server().AddPeer(old)
+	}
+	// Attach to the client and retrieve and interesting metadatas
+	api, err := stack.Attach()
+	if err != nil {
+		stack.Stop()
+		return nil, err
+	}
+	client := ethclient.NewClient(api)
+
+	return &faucet{
+		config:   genesis.Config,
+		stack:    stack,
+		client:   client,
+		index:    index,
+		keystore: ks,
+		account:  ks.Accounts()[0],
+		timeouts: make(map[string]time.Time),
+		update:   make(chan struct{}, 1),
+	}, nil
+}
+
+// close terminates the Ethereum connection and tears down the faucet.
+func (f *faucet) close() error {
+	return f.stack.Stop()
+}
+
+// listenAndServe registers the HTTP handlers for the faucet and boots it up
+// for service user funding requests.
+func (f *faucet) listenAndServe(port int) error {
+	go f.loop()
+
+	http.HandleFunc("/", f.webHandler)
+	http.Handle("/api", websocket.Handler(f.apiHandler))
+
+	return http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
+}
+
+// webHandler handles all non-api requests, simply flattening and returning the
+// faucet website.
+func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write(f.index)
+}
+
+// apiHandler handles requests for Ether grants and transaction statuses.
+func (f *faucet) apiHandler(conn *websocket.Conn) {
+	// Start tracking the connection and drop at the end
+	f.lock.Lock()
+	f.conns = append(f.conns, conn)
+	f.lock.Unlock()
+
+	defer func() {
+		f.lock.Lock()
+		for i, c := range f.conns {
+			if c == conn {
+				f.conns = append(f.conns[:i], f.conns[i+1:]...)
+				break
+			}
+		}
+		f.lock.Unlock()
+	}()
+	// Send a few initial stats to the client
+	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+	websocket.JSON.Send(conn, map[string]interface{}{
+		"funds":    balance.Div(balance, ether),
+		"funded":   nonce,
+		"peers":    f.stack.Server().PeerCount(),
+		"requests": f.reqs,
+	})
+	// Send the initial block to the client
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	header, err := f.client.HeaderByNumber(ctx, nil)
+	cancel()
+
+	if err != nil {
+		log.Error("Failed to retrieve latest header", "err", err)
+	} else {
+		websocket.JSON.Send(conn, header)
+	}
+	// Keep reading requests from the websocket until the connection breaks
+	for {
+		// Fetch the next funding request and validate against github
+		var msg struct {
+			URL     string `json:"url"`
+			Tier    uint   `json:"tier"`
+			Captcha string `json:"captcha"`
+		}
+		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+			return
+		}
+		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
+			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+			continue
+		}
+		if msg.Tier >= uint(*tiersFlag) {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"})
+			continue
+		}
+		log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier)
+
+		// If captcha verifications are enabled, make sure we're not dealing with a robot
+		if *captchaToken != "" {
+			form := url.Values{}
+			form.Add("secret", *captchaSecret)
+			form.Add("response", msg.Captcha)
+
+			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			var result struct {
+				Success bool            `json:"success"`
+				Errors  json.RawMessage `json:"error-codes"`
+			}
+			err = json.NewDecoder(res.Body).Decode(&result)
+			res.Body.Close()
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			if !result.Success {
+				log.Warn("Captcha verification failed", "err", string(result.Errors))
+				websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"})
+				continue
+			}
+		}
+		// Retrieve the gist from the GitHub Gist APIs
+		parts := strings.Split(msg.URL, "/")
+		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+		if *githubUser != "" {
+			req.SetBasicAuth(*githubUser, *githubToken)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		var gist struct {
+			Owner struct {
+				Login string `json:"login"`
+			} `json:"owner"`
+			Files map[string]struct {
+				Content string `json:"content"`
+			} `json:"files"`
+		}
+		err = json.NewDecoder(res.Body).Decode(&gist)
+		res.Body.Close()
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		if gist.Owner.Login == "" {
+			websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"})
+			continue
+		}
+		// Iterate over all the files and look for Ethereum addresses
+		var address common.Address
+		for _, file := range gist.Files {
+			if len(file.Content) == 2+common.AddressLength*2 {
+				address = common.HexToAddress(file.Content)
+			}
+		}
+		if address == (common.Address{}) {
+			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
+			continue
+		}
+		// Validate the user's existence since the API is unhelpful here
+		if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		res.Body.Close()
+
+		if res.StatusCode != 200 {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"})
+			continue
+		}
+		// Ensure the user didn't request funds too recently
+		f.lock.Lock()
+		var (
+			fund    bool
+			timeout time.Time
+		)
+		if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) {
+			// User wasn't funded recently, create the funding transaction
+			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
+			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
+			amount = new(big.Int).Div(amount, new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(msg.Tier)), nil))
+
+			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil)
+			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			// Submit the transaction and mark as funded if successful
+			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			f.reqs = append(f.reqs, &request{
+				Username: gist.Owner.Login,
+				Account:  address,
+				Time:     time.Now(),
+				Tx:       signed,
+			})
+			f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			fund = true
+		}
+		f.lock.Unlock()
+
+		// Send an error if too frequent funding, othewise a success
+		if !fund {
+			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))})
+			continue
+		}
+		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		select {
+		case f.update <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// loop keeps waiting for interesting events and pushes them out to connected
+// websockets.
+func (f *faucet) loop() {
+	// Wait for chain events and push them to clients
+	heads := make(chan *types.Header, 16)
+	sub, err := f.client.SubscribeNewHead(context.Background(), heads)
+	if err != nil {
+		log.Crit("Failed to subscribe to head events", "err", err)
+	}
+	defer sub.Unsubscribe()
+
+	for {
+		select {
+		case head := <-heads:
+			// New chain head arrived, query the current stats and stream to clients
+			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+			balance = new(big.Int).Div(balance, ether)
+
+			price, _ := f.client.SuggestGasPrice(context.Background())
+			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+			f.lock.Lock()
+			f.price, f.nonce = price, nonce
+			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+				f.reqs = f.reqs[1:]
+			}
+			f.lock.Unlock()
+
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{
+					"funds":    balance,
+					"funded":   f.nonce,
+					"peers":    f.stack.Server().PeerCount(),
+					"requests": f.reqs,
+				}); err != nil {
+					log.Warn("Failed to send stats to client", "err", err)
+					conn.Close()
+					continue
+				}
+				if err := websocket.JSON.Send(conn, head); err != nil {
+					log.Warn("Failed to send header to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+
+		case <-f.update:
+			// Pending requests updated, stream to clients
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+					log.Warn("Failed to send requests to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+		}
+	}
+}

cmd/faucet/faucet.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8">
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+
+		<title>{{.Network}}: GitHub Faucet</title>
+
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
+
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-noty/2.4.1/packaged/jquery.noty.packaged.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.18.0/moment.min.js"></script>
+
+		<style>
+			.vertical-center {
+				min-height: 100%;
+				min-height: 100vh;
+				display: flex;
+				align-items: center;
+			}
+			.progress {
+				position: relative;
+			}
+			.progress span {
+				position: absolute;
+				display: block;
+				width: 100%;
+				color: white;
+			 }
+			 pre {
+				 padding: 6px;
+				 margin: 0;
+			 }
+		</style>
+	</head>
+
+	<body>
+		<div class="vertical-center">
+			<div class="container">
+				<div class="row" style="margin-bottom: 16px;">
+					<div class="col-lg-12">
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+					</div>
+				</div>
+				<div class="row">
+					<div class="col-lg-8 col-lg-offset-2">
+						<div class="input-group">
+							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<span class="input-group-btn">
+								<button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Give me Ether	<i class="fa fa-caret-down" aria-hidden="true"></i></button>
+				        <ul class="dropdown-menu dropdown-menu-right">{{range $idx, $amount := .Amounts}}
+				          <li><a style="text-align: center;" onclick="tier={{$idx}}; {{if $.Recaptcha}}grecaptcha.execute(){{else}}submit({{$idx}}){{end}}">{{$amount}} / {{index $.Periods $idx}}</a></li>{{end}}
+				        </ul>
+							</span>
+						</div>{{if .Recaptcha}}
+						<div class="g-recaptcha" data-sitekey="{{.Recaptcha}}" data-callback="submit" data-size="invisible"></div>{{end}}
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+						<div class="col-lg-6 col-lg-offset-3">
+						<div class="panel panel-small panel-default">
+							<div class="panel-body" style="padding: 0; overflow: auto; max-height: 300px;">
+								<table id="requests" class="table table-condensed" style="margin: 0;"></table>
+							</div>
+							<div class="panel-footer">
+								<table style="width: 100%"><tr>
+									<td style="text-align: center;"><i class="fa fa-rss" aria-hidden="true"></i> <span id="peers"></span> peers</td>
+									<td style="text-align: center;"><i class="fa fa-database" aria-hidden="true"></i> <span id="block"></span> blocks</td>
+									<td style="text-align: center;"><i class="fa fa-heartbeat" aria-hidden="true"></i> <span id="funds"></span> Ethers</td>
+									<td style="text-align: center;"><i class="fa fa-university" aria-hidden="true"></i> <span id="funded"></span> funded</td>
+								</tr></table>
+							</div>
+						</div>
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+					<div class="col-lg-12">
+						<h3>How does this work?</h3>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.</p>
+						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+						{{if .Recaptcha}}<em>The faucet is running invisible reCaptcha protection against bots.</em>{{end}}
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<script>
+			// Global variables to hold the current status of the faucet
+			var attempt = 0;
+			var server;
+			var tier = 0;
+
+			// Define the function that submits a gist url to the server
+			var submit = function({{if .Recaptcha}}captcha{{end}}) {
+				server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
+				grecaptcha.reset();{{end}}
+			};
+			// Define a method to reconnect upon server loss
+			var reconnect = function() {
+				if (attempt % 2 == 0) {
+					server = new WebSocket("wss://" + location.host + "/api");
+				} else {
+					server = new WebSocket("ws://" + location.host + "/api");
+				}
+				attempt++;
+
+				server.onmessage = function(event) {
+					var msg = JSON.parse(event.data);
+					if (msg === null) {
+						return;
+					}
+
+					if (msg.funds !== undefined) {
+						$("#funds").text(msg.funds);
+					}
+					if (msg.funded !== undefined) {
+						$("#funded").text(msg.funded);
+					}
+					if (msg.peers !== undefined) {
+						$("#peers").text(msg.peers);
+					}
+					if (msg.number !== undefined) {
+						$("#block").text(parseInt(msg.number, 16));
+					}
+					if (msg.error !== undefined) {
+						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+					}
+					if (msg.success !== undefined) {
+						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+					}
+					if (msg.requests !== undefined && msg.requests !== null) {
+						var content = "";
+						for (var i=0; i<msg.requests.length; i++) {
+							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+						}
+						$("#requests").html("<tbody>" + content + "</tbody>");
+					}
+				}
+				server.onclose = function() { setTimeout(reconnect, 3000); };
+			}
+			// Establish a websocket connection to the API server
+			reconnect();
+		</script>{{if .Recaptcha}}
+		<script src="https://www.google.com/recaptcha/api.js" async defer></script>{{end}}
+	</body>
+</html>

cmd/geth/accountcmd.go

@@ -25,8 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -41,32 +40,39 @@ var (
 
 will prompt for your password and imports your ether presale account.
 It can be used non-interactively with the --password option taking a
-passwordfile as argument containing the wallet password in plaintext.
-
-`,
+passwordfile as argument containing the wallet password in plaintext.`,
 		Subcommands: []cli.Command{
 			{
-				Action:    importWallet,
+
 				Name:      "import",
 				Usage:     "Import Ethereum presale wallet",
 				ArgsUsage: "<keyFile>",
+				Action:    utils.MigrateFlags(importWallet),
+				Category:  "ACCOUNT COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
-TODO: Please write this
-`,
+	geth wallet [options] /path/to/my/presale.wallet
+
+will prompt for your password and imports your ether presale account.
+It can be used non-interactively with the --password option taking a
+passwordfile as argument containing the wallet password in plaintext.`,
 			},
 		},
 	}
-	accountCommand = cli.Command{
-		Action:    accountList,
-		Name:      "account",
-		Usage:     "Manage accounts",
-		ArgsUsage: "",
-		Category:  "ACCOUNT COMMANDS",
-		Description: `
-Manage accounts lets you create new accounts, list all existing accounts,
-import a private key into a new account.
 
-'            help' shows a list of subcommands or help for one subcommand.
+	accountCommand = cli.Command{
+		Name:     "account",
+		Usage:    "Manage accounts",
+		Category: "ACCOUNT COMMANDS",
+		Description: `
+
+Manage accounts, list all existing accounts, import a private key into a new
+account, create a new account or update an existing account.
 
 It supports interactive mode, when you are prompted for password as well as
 non-interactive mode where passwords are supplied via a given password file.
@@ -81,36 +87,34 @@ Note that exporting your key in unencrypted format is NOT supported.
 Keys are stored under <DATADIR>/keystore.
 It is safe to transfer the entire directory or the individual keys therein
 between ethereum nodes by simply copying.
-Make sure you backup your keys regularly.
 
-In order to use your account to send transactions, you need to unlock them using
-the '--unlock' option. The argument is a space separated list of addresses or
-indexes. If used non-interactively with a passwordfile, the file should contain
-the respective passwords one per line. If you unlock n accounts and the password
-file contains less than n entries, then the last password is meant to apply to
-all remaining accounts.
-
-And finally. DO NOT FORGET YOUR PASSWORD.
-`,
+Make sure you backup your keys regularly.`,
 		Subcommands: []cli.Command{
 			{
-				Action:    accountList,
-				Name:      "list",
-				Usage:     "Print account addresses",
-				ArgsUsage: " ",
+				Name:   "list",
+				Usage:  "Print summary of existing accounts",
+				Action: utils.MigrateFlags(accountList),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+				},
 				Description: `
-TODO: Please write this
-`,
+Print a short summary of all accounts`,
 			},
 			{
-				Action:    accountCreate,
-				Name:      "new",
-				Usage:     "Create a new account",
-				ArgsUsage: " ",
+				Name:   "new",
+				Usage:  "Create a new account",
+				Action: utils.MigrateFlags(accountCreate),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     geth account new
 
-Creates a new account. Prints the address.
+Creates a new account and prints the address.
 
 The account is saved in encrypted format, you are prompted for a passphrase.
 
@@ -118,17 +122,20 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    geth --password <passwordfile> account new
-
 Note, this is meant to be used for testing only, it is a bad idea to save your
 password to file or expose in any other way.
 `,
 			},
 			{
-				Action:    accountUpdate,
 				Name:      "update",
 				Usage:     "Update an existing account",
+				Action:    utils.MigrateFlags(accountUpdate),
 				ArgsUsage: "<address>",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     geth account update <address>
 
@@ -142,16 +149,22 @@ format to the newest format or change the password for an account.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    geth --password <passwordfile> account update <address>
+    geth account update [options] <address>
 
 Since only one password can be given, only format update can be performed,
 changing your password is only possible interactively.
 `,
 			},
 			{
-				Action:    accountImport,
-				Name:      "import",
-				Usage:     "Import a private key into a new account",
+				Name:   "import",
+				Usage:  "Import a private key into a new account",
+				Action: utils.MigrateFlags(accountImport),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				ArgsUsage: "<keyFile>",
 				Description: `
     geth account import <keyfile>
@@ -167,7 +180,7 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the -password flag:
 
-    geth --password <passwordfile> account import <keyfile>
+    geth account import [options] <keyfile>
 
 Note:
 As you can directly copy your encrypted accounts to another ethereum instance,
@@ -180,8 +193,7 @@ nodes.
 )
 
 func accountList(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-
+	stack, _ := makeConfigNode(ctx)
 	var index int
 	for _, wallet := range stack.AccountManager().Wallets() {
 		for _, account := range wallet.Accounts() {
@@ -203,11 +215,11 @@ func unlockAccount(ctx *cli.Context, ks *keystore.KeyStore, address string, i in
 		password := getPassPhrase(prompt, false, i, passwords)
 		err = ks.Unlock(account, password)
 		if err == nil {
-			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
+			log.Info("Unlocked account", "address", account.Address.Hex())
 			return account, password
 		}
 		if err, ok := err.(*keystore.AmbiguousAddrError); ok {
-			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
+			log.Info("Unlocked account", "address", account.Address.Hex())
 			return ambiguousAddrRecovery(ks, err, password), password
 		}
 		if err != keystore.ErrDecrypt {
@@ -217,6 +229,7 @@ func unlockAccount(ctx *cli.Context, ks *keystore.KeyStore, address string, i in
 	}
 	// All trials expended to unlock account, bail out
 	utils.Fatalf("Failed to unlock account %s (%v)", address, err)
+
 	return accounts.Account{}, ""
 }
 
@@ -278,7 +291,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -296,13 +309,15 @@ func accountUpdate(ctx *cli.Context) error {
 	if len(ctx.Args()) == 0 {
 		utils.Fatalf("No accounts specified to update")
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	account, oldPassword := unlockAccount(ctx, ks, ctx.Args().First(), 0, nil)
-	newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
-	if err := ks.Update(account, oldPassword, newPassword); err != nil {
-		utils.Fatalf("Could not update the account: %v", err)
+	for _, addr := range ctx.Args() {
+		account, oldPassword := unlockAccount(ctx, ks, addr, 0, nil)
+		newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
+		if err := ks.Update(account, oldPassword, newPassword); err != nil {
+			utils.Fatalf("Could not update the account: %v", err)
+		}
 	}
 	return nil
 }
@@ -317,7 +332,7 @@ func importWallet(ctx *cli.Context) error {
 		utils.Fatalf("Could not read wallet file: %v", err)
 	}
 
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -338,7 +353,7 @@ func accountImport(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("Failed to load the private key: %v", err)
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)

cmd/geth/accountcmd_test.go

@@ -43,13 +43,13 @@ func tmpDatadirWithKeystore(t *testing.T) string {
 }
 
 func TestAccountListEmpty(t *testing.T) {
-	geth := runGeth(t, "account")
+	geth := runGeth(t, "account", "list")
 	geth.expectExit()
 }
 
 func TestAccountList(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t, "--datadir", datadir, "account")
+	geth := runGeth(t, "account", "list", "--datadir", datadir)
 	defer geth.expectExit()
 	if runtime.GOOS == "windows" {
 		geth.expect(`
@@ -67,7 +67,7 @@ Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/k
 }
 
 func TestAccountNew(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "account", "new")
+	geth := runGeth(t, "account", "new", "--lightkdf")
 	defer geth.expectExit()
 	geth.expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
@@ -79,7 +79,7 @@ Repeat passphrase: {{.InputLine "foobar"}}
 }
 
 func TestAccountNewBadRepeat(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "account", "new")
+	geth := runGeth(t, "account", "new", "--lightkdf")
 	defer geth.expectExit()
 	geth.expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
@@ -92,9 +92,9 @@ Fatal: Passphrases do not match
 
 func TestAccountUpdate(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
+	geth := runGeth(t, "account", "update",
 		"--datadir", datadir, "--lightkdf",
-		"account", "update", "f466859ead1932d743d622cb74fc058882e8648a")
+		"f466859ead1932d743d622cb74fc058882e8648a")
 	defer geth.expectExit()
 	geth.expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
@@ -107,7 +107,7 @@ Repeat passphrase: {{.InputLine "foobar2"}}
 }
 
 func TestWalletImport(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
+	geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
 	defer geth.expectExit()
 	geth.expect(`
 !! Unsupported terminal, password will be echoed.
@@ -122,7 +122,7 @@ Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
 }
 
 func TestWalletImportBadPassword(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
+	geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
 	defer geth.expectExit()
 	geth.expect(`
 !! Unsupported terminal, password will be echoed.
@@ -145,7 +145,8 @@ Passphrase: {{.InputLine "foobar"}}
 	geth.expectExit()
 
 	wantMessages := []string{
-		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
+		"Unlocked account",
+		"=0xf466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
 		if !strings.Contains(geth.stderrText(), m) {
@@ -189,8 +190,9 @@ Passphrase: {{.InputLine "foobar"}}
 	geth.expectExit()
 
 	wantMessages := []string{
-		"Unlocked account 7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
-		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
+		"Unlocked account",
+		"=0x7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+		"=0x289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
 		if !strings.Contains(geth.stderrText(), m) {
@@ -208,8 +210,9 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 	geth.expectExit()
 
 	wantMessages := []string{
-		"Unlocked account 7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
-		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
+		"Unlocked account",
+		"=0x7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+		"=0x289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
 		if !strings.Contains(geth.stderrText(), m) {
@@ -257,7 +260,8 @@ In order to avoid this warning, you need to remove the following duplicate key f
 	geth.expectExit()
 
 	wantMessages := []string{
-		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
+		"Unlocked account",
+		"=0xf466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
 		if !strings.Contains(geth.stderrText(), m) {

cmd/geth/bugcmd.go

@@ -0,0 +1,109 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/cmd/internal/browser"
+	"github.com/ethereum/go-ethereum/params"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var bugCommand = cli.Command{
+	Action:    utils.MigrateFlags(reportBug),
+	Name:      "bug",
+	Usage:     "opens a window to report a bug on the geth repo",
+	ArgsUsage: " ",
+	Category:  "MISCELLANEOUS COMMANDS",
+}
+
+const issueUrl = "https://github.com/ethereum/go-ethereum/issues/new"
+
+// reportBug reports a bug by opening a new URL to the go-ethereum GH issue
+// tracker and setting default values as the issue body.
+func reportBug(ctx *cli.Context) error {
+	// execute template and write contents to buff
+	var buff bytes.Buffer
+
+	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "Version:", params.Version)
+	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
+	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
+	printOSDetails(&buff)
+
+	// open a new GH issue
+	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
+		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+	}
+	return nil
+}
+
+// copied from the Go source. Copyright 2017 The Go Authors
+func printOSDetails(w io.Writer) {
+	switch runtime.GOOS {
+	case "darwin":
+		printCmdOut(w, "uname -v: ", "uname", "-v")
+		printCmdOut(w, "", "sw_vers")
+	case "linux":
+		printCmdOut(w, "uname -sr: ", "uname", "-sr")
+		printCmdOut(w, "", "lsb_release", "-a")
+	case "openbsd", "netbsd", "freebsd", "dragonfly":
+		printCmdOut(w, "uname -v: ", "uname", "-v")
+	case "solaris":
+		out, err := ioutil.ReadFile("/etc/release")
+		if err == nil {
+			fmt.Fprintf(w, "/etc/release: %s\n", out)
+		} else {
+			fmt.Printf("failed to read /etc/release: %v\n", err)
+		}
+	}
+}
+
+// printCmdOut prints the output of running the given command.
+// It ignores failures; 'go bug' is best effort.
+//
+// copied from the Go source. Copyright 2017 The Go Authors
+func printCmdOut(w io.Writer, prefix, path string, args ...string) {
+	cmd := exec.Command(path, args...)
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err)
+		return
+	}
+	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
+}
+
+const header = `Please answer these questions before submitting your issue. Thanks!
+
+#### What did you do?
+ 
+#### What did you expect to see?
+ 
+#### What did you see instead?
+ 
+#### System details
+`

cmd/geth/chaincmd.go

@@ -17,9 +17,9 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"sync/atomic"
@@ -32,8 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
@@ -41,100 +40,123 @@ import (
 
 var (
 	initCommand = cli.Command{
-		Action:    initGenesis,
+		Action:    utils.MigrateFlags(initGenesis),
 		Name:      "init",
 		Usage:     "Bootstrap and initialize a new genesis block",
 		ArgsUsage: "<genesisPath>",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
 participating.
-`,
+
+It expects the genesis file as argument.`,
 	}
 	importCommand = cli.Command{
-		Action:    importChain,
+		Action:    utils.MigrateFlags(importChain),
 		Name:      "import",
 		Usage:     "Import a blockchain file",
-		ArgsUsage: "<filename>",
-		Category:  "BLOCKCHAIN COMMANDS",
+		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-TODO: Please write this
-`,
+The import command imports blocks from an RLP-encoded form. The form can be one file
+with several RLP-encoded blocks, or several files can be used.
+
+If only one file is used, import error will result in failure. If several files are used, 
+processing will proceed even if an individual RLP-file import failure occurs.`,
 	}
 	exportCommand = cli.Command{
-		Action:    exportChain,
+		Action:    utils.MigrateFlags(exportChain),
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
-if already existing.
-`,
-	}
-	upgradedbCommand = cli.Command{
-		Action:    upgradeDB,
-		Name:      "upgradedb",
-		Usage:     "Upgrade chainblock database",
-		ArgsUsage: " ",
-		Category:  "BLOCKCHAIN COMMANDS",
-		Description: `
-TODO: Please write this
-`,
+if already existing.`,
 	}
 	removedbCommand = cli.Command{
-		Action:    removeDB,
+		Action:    utils.MigrateFlags(removeDB),
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: " ",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-TODO: Please write this
-`,
+Remove blockchain and state databases`,
 	}
 	dumpCommand = cli.Command{
-		Action:    dump,
+		Action:    utils.MigrateFlags(dump),
 		Name:      "dump",
 		Usage:     "Dump a specific block from storage",
 		ArgsUsage: "[<blockHash> | <blockNum>]...",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The arguments are interpreted as block numbers or hashes.
-Use "ethereum dump 0" to dump the genesis block.
-`,
+Use "ethereum dump 0" to dump the genesis block.`,
 	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
 // the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
 func initGenesis(ctx *cli.Context) error {
+	// Make sure we have a valid genesis JSON
 	genesisPath := ctx.Args().First()
 	if len(genesisPath) == 0 {
-		utils.Fatalf("must supply path to genesis JSON file")
+		utils.Fatalf("Must supply path to genesis JSON file")
 	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
 
+	genesis := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(genesis); err != nil {
+		utils.Fatalf("invalid genesis file: %v", err)
+	}
+	// Open an initialise both full and light databases
 	stack := makeFullNode(ctx)
-	chaindb := utils.MakeChainDatabase(ctx, stack)
-
-	genesisFile, err := os.Open(genesisPath)
-	if err != nil {
-		utils.Fatalf("failed to read genesis file: %v", err)
+	for _, name := range []string{"chaindata", "lightchaindata"} {
+		chaindb, err := stack.OpenDatabase(name, 0, 0)
+		if err != nil {
+			utils.Fatalf("Failed to open database: %v", err)
+		}
+		_, hash, err := core.SetupGenesisBlock(chaindb, genesis)
+		if err != nil {
+			utils.Fatalf("Failed to write genesis block: %v", err)
+		}
+		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
-	defer genesisFile.Close()
-
-	block, err := core.WriteGenesisBlock(chaindb, genesisFile)
-	if err != nil {
-		utils.Fatalf("failed to write genesis block: %v", err)
-	}
-	glog.V(logger.Info).Infof("successfully wrote genesis block and/or chain rule set: %x", block.Hash())
 	return nil
 }
 
 func importChain(ctx *cli.Context) error {
-	if len(ctx.Args()) != 1 {
+	if len(ctx.Args()) < 1 {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
@@ -158,9 +180,19 @@ func importChain(ctx *cli.Context) error {
 	}()
 	// Import the chain
 	start := time.Now()
-	if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
-		utils.Fatalf("Import error: %v", err)
+
+	if len(ctx.Args()) == 1 {
+		if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
+			utils.Fatalf("Import error: %v", err)
+		}
+	} else {
+		for _, arg := range ctx.Args() {
+			if err := utils.ImportChain(chain, arg); err != nil {
+				log.Error("Import error", "file", arg, "err", err)
+			}
+		}
 	}
+
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing
@@ -183,6 +215,10 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Allocations:   %.3f million\n", float64(mem.Mallocs)/1000000)
 	fmt.Printf("GC pause:      %v\n\n", time.Duration(mem.PauseTotalNs))
 
+	if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
+		return nil
+	}
+
 	// Compact the entire database to more accurately measure disk io and print the stats
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
@@ -233,72 +269,34 @@ func exportChain(ctx *cli.Context) error {
 }
 
 func removeDB(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	dbdir := stack.ResolvePath(utils.ChainDbName(ctx))
-	if !common.FileExist(dbdir) {
-		fmt.Println(dbdir, "does not exist")
-		return nil
-	}
+	stack, _ := makeConfigNode(ctx)
 
-	fmt.Println(dbdir)
-	confirm, err := console.Stdin.PromptConfirm("Remove this database?")
-	switch {
-	case err != nil:
-		utils.Fatalf("%v", err)
-	case !confirm:
-		fmt.Println("Operation aborted")
-	default:
-		fmt.Println("Removing...")
-		start := time.Now()
-		os.RemoveAll(dbdir)
-		fmt.Printf("Removed in %v\n", time.Since(start))
+	for _, name := range []string{"chaindata", "lightchaindata"} {
+		// Ensure the database exists in the first place
+		logger := log.New("database", name)
+
+		dbdir := stack.ResolvePath(name)
+		if !common.FileExist(dbdir) {
+			logger.Info("Database doesn't exist, skipping", "path", dbdir)
+			continue
+		}
+		// Confirm removal and execute
+		fmt.Println(dbdir)
+		confirm, err := console.Stdin.PromptConfirm("Remove this database?")
+		switch {
+		case err != nil:
+			utils.Fatalf("%v", err)
+		case !confirm:
+			logger.Warn("Database deletion aborted")
+		default:
+			start := time.Now()
+			os.RemoveAll(dbdir)
+			logger.Info("Database successfully deleted", "elapsed", common.PrettyDuration(time.Since(start)))
+		}
 	}
 	return nil
 }
 
-func upgradeDB(ctx *cli.Context) error {
-	glog.Infoln("Upgrading blockchain database")
-
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	chain, chainDb := utils.MakeChain(ctx, stack)
-	bcVersion := core.GetBlockChainVersion(chainDb)
-	if bcVersion == 0 {
-		bcVersion = core.BlockChainVersion
-	}
-
-	// Export the current chain.
-	filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405"))
-	exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename)
-	if err := utils.ExportChain(chain, exportFile); err != nil {
-		utils.Fatalf("Unable to export chain for reimport %s", err)
-	}
-	chainDb.Close()
-	if dir := dbDirectory(chainDb); dir != "" {
-		os.RemoveAll(dir)
-	}
-
-	// Import the chain file.
-	chain, chainDb = utils.MakeChain(ctx, stack)
-	core.WriteBlockChainVersion(chainDb, core.BlockChainVersion)
-	err := utils.ImportChain(chain, exportFile)
-	chainDb.Close()
-	if err != nil {
-		utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile)
-	} else {
-		os.Remove(exportFile)
-		glog.Infoln("Import finished")
-	}
-	return nil
-}
-
-func dbDirectory(db ethdb.Database) string {
-	ldb, ok := db.(*ethdb.LDBDatabase)
-	if !ok {
-		return ""
-	}
-	return ldb.Path()
-}
-
 func dump(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
 	chain, chainDb := utils.MakeChain(ctx, stack)
@@ -330,9 +328,3 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
-
-func closeAll(dbs ...ethdb.Database) {
-	for _, db := range dbs {
-		db.Close()
-	}
-}

cmd/geth/config.go

@@ -0,0 +1,187 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"unicode"
+
+	cli "gopkg.in/urfave/cli.v1"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/contracts/release"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/naoina/toml"
+)
+
+var (
+	dumpConfigCommand = cli.Command{
+		Action:      utils.MigrateFlags(dumpConfig),
+		Name:        "dumpconfig",
+		Usage:       "Show configuration values",
+		ArgsUsage:   "",
+		Flags:       append(nodeFlags, rpcFlags...),
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: `The dumpconfig command shows configuration values.`,
+	}
+
+	configFileFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "TOML configuration file",
+	}
+)
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}
+
+type ethstatsConfig struct {
+	URL string `toml:",omitempty"`
+}
+
+type gethConfig struct {
+	Eth      eth.Config
+	Node     node.Config
+	Ethstats ethstatsConfig
+}
+
+func loadConfig(file string, cfg *gethConfig) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(cfg)
+	// Add file name to errors that have a line number.
+	if _, ok := err.(*toml.LineError); ok {
+		err = errors.New(file + ", " + err.Error())
+	}
+	return err
+}
+
+func defaultNodeConfig() node.Config {
+	cfg := node.DefaultConfig
+	cfg.Name = clientIdentifier
+	cfg.Version = params.VersionWithCommit(gitCommit)
+	cfg.HTTPModules = append(cfg.HTTPModules, "eth")
+	cfg.WSModules = append(cfg.WSModules, "eth")
+	cfg.IPCPath = "geth.ipc"
+	return cfg
+}
+
+func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
+	// Load defaults.
+	cfg := gethConfig{
+		Eth:  eth.DefaultConfig,
+		Node: defaultNodeConfig(),
+	}
+
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+
+	// Apply flags.
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	stack, err := node.New(&cfg.Node)
+	if err != nil {
+		utils.Fatalf("Failed to create the protocol stack: %v", err)
+	}
+	utils.SetEthConfig(ctx, stack, &cfg.Eth)
+	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
+		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
+	}
+
+	return stack, cfg
+}
+
+func makeFullNode(ctx *cli.Context) *node.Node {
+	stack, cfg := makeConfigNode(ctx)
+
+	utils.RegisterEthService(stack, &cfg.Eth)
+
+	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
+	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	if shhEnabled || shhAutoEnabled {
+		utils.RegisterShhService(stack)
+	}
+
+	// Add the Ethereum Stats daemon if requested.
+	if cfg.Ethstats.URL != "" {
+		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
+	}
+
+	// Add the release oracle service so it boots along with node.
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		config := release.Config{
+			Oracle: relOracle,
+			Major:  uint32(params.VersionMajor),
+			Minor:  uint32(params.VersionMinor),
+			Patch:  uint32(params.VersionPatch),
+		}
+		commit, _ := hex.DecodeString(gitCommit)
+		copy(config.Commit[:], commit)
+		return release.NewReleaseService(ctx, config)
+	}); err != nil {
+		utils.Fatalf("Failed to register the Geth release oracle service: %v", err)
+	}
+	return stack
+}
+
+// dumpConfig is the dumpconfig command.
+func dumpConfig(ctx *cli.Context) error {
+	_, cfg := makeConfigNode(ctx)
+	comment := ""
+
+	if cfg.Eth.Genesis != nil {
+		cfg.Eth.Genesis = nil
+		comment += "# Note: this config doesn't contain the genesis block.\n\n"
+	}
+
+	out, err := tomlSettings.Marshal(&cfg)
+	if err != nil {
+		return err
+	}
+	io.WriteString(os.Stdout, comment)
+	os.Stdout.Write(out)
+	return nil
+}

cmd/geth/consolecmd.go

@@ -29,41 +29,44 @@ import (
 )
 
 var (
+	consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag}
+
 	consoleCommand = cli.Command{
-		Action:    localConsole,
-		Name:      "console",
-		Usage:     "Start an interactive JavaScript environment",
-		ArgsUsage: "", // TODO: Write this!
-		Category:  "CONSOLE COMMANDS",
+		Action:   utils.MigrateFlags(localConsole),
+		Name:     "console",
+		Usage:    "Start an interactive JavaScript environment",
+		Flags:    append(append(nodeFlags, rpcFlags...), consoleFlags...),
+		Category: "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console
-`,
+See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`,
 	}
+
 	attachCommand = cli.Command{
-		Action:    remoteConsole,
+		Action:    utils.MigrateFlags(remoteConsole),
 		Name:      "attach",
 		Usage:     "Start an interactive JavaScript environment (connect to node)",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "[endpoint]",
+		Flags:     append(consoleFlags, utils.DataDirFlag),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
 See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.
-This command allows to open a console on a running geth node.
-`,
+This command allows to open a console on a running geth node.`,
 	}
+
 	javascriptCommand = cli.Command{
-		Action:    ephemeralConsole,
+		Action:    utils.MigrateFlags(ephemeralConsole),
 		Name:      "js",
 		Usage:     "Execute the specified JavaScript files",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "<jsfile> [jsfile...]",
+		Flags:     append(nodeFlags, consoleFlags...),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
-JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console
-`,
+JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console`,
 	}
 )
 
@@ -81,11 +84,12 @@ func localConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc geth: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
@@ -118,17 +122,18 @@ func remoteConsole(ctx *cli.Context) error {
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
 	}
 	defer console.Stop(false)
 
-	// If only a short execution was requested, evaluate and return
 	if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" {
 		console.Evaluate(script)
 		return nil
 	}
+
 	// Otherwise print the welcome screen and enter interactive mode
 	console.Welcome()
 	console.Interactive()
@@ -151,7 +156,7 @@ func dialRPC(endpoint string) (*rpc.Client, error) {
 }
 
 // ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript
-// console to it, and each of the files specified as arguments and tears the
+// console to it, executes each of the files specified as arguments and tears
 // everything down.
 func ephemeralConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
@@ -165,11 +170,12 @@ func ephemeralConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc geth: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)

cmd/geth/consolecmd_test.go

@@ -22,14 +22,17 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rpc"
+)
+
+const (
+	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
+	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
 // Tests that a node embedded within a console can be started up properly and
@@ -45,24 +48,21 @@ func TestConsoleWelcome(t *testing.T) {
 
 	// Gather all the infos the welcome message needs to contain
 	geth.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	geth.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	geth.setTemplateFunc("gover", runtime.Version)
 	geth.setTemplateFunc("gethver", func() string { return params.Version })
 	geth.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
-	geth.setTemplateFunc("apis", func() []string {
-		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		sort.Strings(apis)
-		return apis
-	})
+	geth.setTemplateFunc("apis", func() string { return ipcAPIs })
 
 	// Verify the actual welcome message to the required template
 	geth.expect(`
 Welcome to the Geth JavaScript console!
 
-instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{.Etherbase}}
 at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit"}}
 `)
@@ -88,7 +88,7 @@ func TestIPCAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "ipc:"+ipc)
+	testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
 
 	geth.interrupt()
 	geth.expectExit()
@@ -102,7 +102,7 @@ func TestHTTPAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--rpc", "--rpcport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "http://localhost:"+port)
+	testAttachWelcome(t, geth, "http://localhost:"+port, httpAPIs)
 
 	geth.interrupt()
 	geth.expectExit()
@@ -117,13 +117,13 @@ func TestWSAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--ws", "--wsport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "ws://localhost:"+port)
+	testAttachWelcome(t, geth, "ws://localhost:"+port, httpAPIs)
 
 	geth.interrupt()
 	geth.expectExit()
 }
 
-func testAttachWelcome(t *testing.T, geth *testgeth, endpoint string) {
+func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
 	// Attach to a running geth note and terminate immediately
 	attach := runGeth(t, "attach", endpoint)
 	defer attach.expectExit()
@@ -131,32 +131,24 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint string) {
 
 	// Gather all the infos the welcome message needs to contain
 	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	attach.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	attach.setTemplateFunc("gover", runtime.Version)
 	attach.setTemplateFunc("gethver", func() string { return params.Version })
 	attach.setTemplateFunc("etherbase", func() string { return geth.Etherbase })
 	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
 	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
 	attach.setTemplateFunc("datadir", func() string { return geth.Datadir })
-	attach.setTemplateFunc("apis", func() []string {
-		var apis []string
-		if strings.HasPrefix(endpoint, "ipc") {
-			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		} else {
-			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
-		}
-		sort.Strings(apis)
-		return apis
-	})
+	attach.setTemplateFunc("apis", func() string { return apis })
 
 	// Verify the actual welcome message to the required template
 	attach.expect(`
 Welcome to the Geth JavaScript console!
 
-instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{etherbase}}
 at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit" }}
 `)

cmd/geth/dao_test.go

@@ -84,27 +84,24 @@ var daoGenesisForkBlock = big.NewInt(314)
 // set in the database after various initialization procedures and invocations.
 func TestDAOForkBlockNewChain(t *testing.T) {
 	for i, arg := range []struct {
-		testnet     bool
 		genesis     string
 		expectBlock *big.Int
 		expectVote  bool
 	}{
 		// Test DAO Default Mainnet
-		{false, "", params.MainNetDAOForkBlock, true},
-		// test DAO Default Testnet
-		{true, "", params.TestNetDAOForkBlock, true},
+		{"", params.MainNetDAOForkBlock, true},
 		// test DAO Init Old Privnet
-		{false, daoOldGenesis, nil, false},
+		{daoOldGenesis, nil, false},
 		// test DAO Default No Fork Privnet
-		{false, daoNoForkGenesis, daoGenesisForkBlock, false},
+		{daoNoForkGenesis, daoGenesisForkBlock, false},
 		// test DAO Default Pro Fork Privnet
-		{false, daoProForkGenesis, daoGenesisForkBlock, true},
+		{daoProForkGenesis, daoGenesisForkBlock, true},
 	} {
-		testDAOForkBlockNewChain(t, i, arg.testnet, arg.genesis, arg.expectBlock, arg.expectVote)
+		testDAOForkBlockNewChain(t, i, arg.genesis, arg.expectBlock, arg.expectVote)
 	}
 }
 
-func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis string, expectBlock *big.Int, expectVote bool) {
+func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
 	datadir := tmpdir(t)
 	defer os.RemoveAll(datadir)
@@ -119,17 +116,11 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	} else {
 		// Force chain initialization
 		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
-		if testnet {
-			args = append(args, "--testnet")
-		}
 		geth := runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...)
 		geth.cmd.Wait()
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "geth", "chaindata")
-	if testnet && genesis == "" {
-		path = filepath.Join(datadir, "testnet", "geth", "chaindata")
-	}
 	db, err := ethdb.NewLDBDatabase(path, 0, 0)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
@@ -137,9 +128,6 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	defer db.Close()
 
 	genesisHash := common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
-	if testnet {
-		genesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
-	}
 	if genesis != "" {
 		genesisHash = daoGenesisHash
 	}

cmd/geth/main.go

@@ -18,7 +18,6 @@
 package main
 
 import (
-	"encoding/hex"
 	"fmt"
 	"os"
 	"runtime"
@@ -30,16 +29,12 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
-	"github.com/ethereum/go-ethereum/contracts/release"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -54,19 +49,94 @@ var (
 	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
+	// flags that configure the node
+	nodeFlags = []cli.Flag{
+		utils.IdentityFlag,
+		utils.UnlockedAccountFlag,
+		utils.PasswordFileFlag,
+		utils.BootnodesFlag,
+		utils.BootnodesV4Flag,
+		utils.BootnodesV5Flag,
+		utils.DataDirFlag,
+		utils.KeyStoreDirFlag,
+		utils.NoUSBFlag,
+		utils.EthashCacheDirFlag,
+		utils.EthashCachesInMemoryFlag,
+		utils.EthashCachesOnDiskFlag,
+		utils.EthashDatasetDirFlag,
+		utils.EthashDatasetsInMemoryFlag,
+		utils.EthashDatasetsOnDiskFlag,
+		utils.TxPoolPriceLimitFlag,
+		utils.TxPoolPriceBumpFlag,
+		utils.TxPoolAccountSlotsFlag,
+		utils.TxPoolGlobalSlotsFlag,
+		utils.TxPoolAccountQueueFlag,
+		utils.TxPoolGlobalQueueFlag,
+		utils.TxPoolLifetimeFlag,
+		utils.FastSyncFlag,
+		utils.LightModeFlag,
+		utils.SyncModeFlag,
+		utils.LightServFlag,
+		utils.LightPeersFlag,
+		utils.LightKDFFlag,
+		utils.CacheFlag,
+		utils.TrieCacheGenFlag,
+		utils.ListenPortFlag,
+		utils.MaxPeersFlag,
+		utils.MaxPendingPeersFlag,
+		utils.EtherbaseFlag,
+		utils.GasPriceFlag,
+		utils.MinerThreadsFlag,
+		utils.MiningEnabledFlag,
+		utils.TargetGasLimitFlag,
+		utils.NATFlag,
+		utils.NoDiscoverFlag,
+		utils.DiscoveryV5Flag,
+		utils.NetrestrictFlag,
+		utils.NodeKeyFileFlag,
+		utils.NodeKeyHexFlag,
+		utils.WhisperEnabledFlag,
+		utils.DevModeFlag,
+		utils.TestnetFlag,
+		utils.RinkebyFlag,
+		utils.VMEnableDebugFlag,
+		utils.NetworkIdFlag,
+		utils.RPCCORSDomainFlag,
+		utils.EthStatsURLFlag,
+		utils.MetricsEnabledFlag,
+		utils.FakePoWFlag,
+		utils.NoCompactionFlag,
+		utils.GpoBlocksFlag,
+		utils.GpoPercentileFlag,
+		utils.ExtraDataFlag,
+		configFileFlag,
+	}
+
+	rpcFlags = []cli.Flag{
+		utils.RPCEnabledFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCPortFlag,
+		utils.RPCApiFlag,
+		utils.WSEnabledFlag,
+		utils.WSListenAddrFlag,
+		utils.WSPortFlag,
+		utils.WSApiFlag,
+		utils.WSAllowedOriginsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+	}
 )
 
 func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2017 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
 		importCommand,
 		exportCommand,
-		upgradedbCommand,
 		removedbCommand,
 		dumpCommand,
 		// See monitorcmd.go:
@@ -81,74 +151,15 @@ func init() {
 		// See misccmd.go:
 		makedagCommand,
 		versionCommand,
+		bugCommand,
 		licenseCommand,
+		// See config.go
+		dumpConfigCommand,
 	}
 
-	app.Flags = []cli.Flag{
-		utils.IdentityFlag,
-		utils.UnlockedAccountFlag,
-		utils.PasswordFileFlag,
-		utils.BootnodesFlag,
-		utils.DataDirFlag,
-		utils.KeyStoreDirFlag,
-		utils.FastSyncFlag,
-		utils.LightModeFlag,
-		utils.LightServFlag,
-		utils.LightPeersFlag,
-		utils.LightKDFFlag,
-		utils.CacheFlag,
-		utils.TrieCacheGenFlag,
-		utils.JSpathFlag,
-		utils.ListenPortFlag,
-		utils.MaxPeersFlag,
-		utils.MaxPendingPeersFlag,
-		utils.EtherbaseFlag,
-		utils.GasPriceFlag,
-		utils.MinerThreadsFlag,
-		utils.MiningEnabledFlag,
-		utils.AutoDAGFlag,
-		utils.TargetGasLimitFlag,
-		utils.NATFlag,
-		utils.NoDiscoverFlag,
-		utils.DiscoveryV5Flag,
-		utils.NetrestrictFlag,
-		utils.NodeKeyFileFlag,
-		utils.NodeKeyHexFlag,
-		utils.RPCEnabledFlag,
-		utils.RPCListenAddrFlag,
-		utils.RPCPortFlag,
-		utils.RPCApiFlag,
-		utils.WSEnabledFlag,
-		utils.WSListenAddrFlag,
-		utils.WSPortFlag,
-		utils.WSApiFlag,
-		utils.WSAllowedOriginsFlag,
-		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
-		utils.IPCPathFlag,
-		utils.ExecFlag,
-		utils.PreloadJSFlag,
-		utils.WhisperEnabledFlag,
-		utils.DevModeFlag,
-		utils.TestNetFlag,
-		utils.VMForceJitFlag,
-		utils.VMJitCacheFlag,
-		utils.VMEnableJitFlag,
-		utils.VMEnableDebugFlag,
-		utils.NetworkIdFlag,
-		utils.RPCCORSDomainFlag,
-		utils.EthStatsURLFlag,
-		utils.MetricsEnabledFlag,
-		utils.FakePoWFlag,
-		utils.SolcPathFlag,
-		utils.GpoMinGasPriceFlag,
-		utils.GpoMaxGasPriceFlag,
-		utils.GpoFullBlockRatioFlag,
-		utils.GpobaseStepDownFlag,
-		utils.GpobaseStepUpFlag,
-		utils.GpobaseCorrectionFactorFlag,
-		utils.ExtraDataFlag,
-	}
+	app.Flags = append(app.Flags, nodeFlags...)
+	app.Flags = append(app.Flags, rpcFlags...)
+	app.Flags = append(app.Flags, consoleFlags...)
 	app.Flags = append(app.Flags, debug.Flags...)
 
 	app.Before = func(ctx *cli.Context) error {
@@ -159,12 +170,6 @@ func init() {
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		// This should be the only place where reporting is enabled
-		// because it is not intended to run while testing.
-		// In addition to this check, bad block reports are sent only
-		// for chains with the main network genesis block and network id 1.
-		eth.EnableBadBlockReporting = true
-
 		utils.SetupNetwork(ctx)
 		return nil
 	}
@@ -193,53 +198,6 @@ func geth(ctx *cli.Context) error {
 	return nil
 }
 
-func makeFullNode(ctx *cli.Context) *node.Node {
-	// Create the default extradata and construct the base node
-	var clientInfo = struct {
-		Version   uint
-		Name      string
-		GoVersion string
-		Os        string
-	}{uint(params.VersionMajor<<16 | params.VersionMinor<<8 | params.VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
-	extra, err := rlp.EncodeToBytes(clientInfo)
-	if err != nil {
-		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
-	}
-	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
-		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
-		glog.V(logger.Debug).Infof("extra: %x\n", extra)
-		extra = nil
-	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	utils.RegisterEthService(ctx, stack, extra)
-
-	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
-	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
-	if shhEnabled || shhAutoEnabled {
-		utils.RegisterShhService(stack)
-	}
-	// Add the Ethereum Stats daemon if requested
-	if url := ctx.GlobalString(utils.EthStatsURLFlag.Name); url != "" {
-		utils.RegisterEthStatsService(stack, url)
-	}
-	// Add the release oracle service so it boots along with node.
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		config := release.Config{
-			Oracle: relOracle,
-			Major:  uint32(params.VersionMajor),
-			Minor:  uint32(params.VersionMinor),
-			Patch:  uint32(params.VersionPatch),
-		}
-		commit, _ := hex.DecodeString(gitCommit)
-		copy(config.Commit[:], commit)
-		return release.NewReleaseService(ctx, config)
-	}); err != nil {
-		utils.Fatalf("Failed to register the Geth release oracle service: %v", err)
-	}
-	return stack
-}
-
 // startNode boots up the system node and all registered protocols, after which
 // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the
 // miner.
@@ -272,7 +230,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		// Open and self derive any wallets already attached
 		for _, wallet := range stack.AccountManager().Wallets() {
 			if err := wallet.Open(""); err != nil {
-				glog.V(logger.Warn).Infof("Failed to open wallet %s: %v", wallet.URL(), err)
+				log.Warn("Failed to open wallet", "url", wallet.URL(), "err", err)
 			} else {
 				wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
 			}
@@ -281,13 +239,13 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		for event := range events {
 			if event.Arrive {
 				if err := event.Wallet.Open(""); err != nil {
-					glog.V(logger.Info).Infof("New wallet appeared: %s, failed to open: %s", event.Wallet.URL(), err)
+					log.Warn("New wallet appeared, failed to open", "url", event.Wallet.URL(), "err", err)
 				} else {
-					glog.V(logger.Info).Infof("New wallet appeared: %s, %s", event.Wallet.URL(), event.Wallet.Status())
+					log.Info("New wallet appeared", "url", event.Wallet.URL(), "status", event.Wallet.Status())
 					event.Wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
 				}
 			} else {
-				glog.V(logger.Info).Infof("Old wallet dropped:  %s", event.Wallet.URL())
+				log.Info("Old wallet dropped", "url", event.Wallet.URL())
 				event.Wallet.Close()
 			}
 		}
@@ -298,7 +256,15 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("ethereum service not running: %v", err)
 		}
-		if err := ethereum.StartMining(ctx.GlobalInt(utils.MinerThreadsFlag.Name)); err != nil {
+		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
+			type threaded interface {
+				SetThreads(threads int)
+			}
+			if th, ok := ethereum.Engine().(threaded); ok {
+				th.SetThreads(threads)
+			}
+		}
+		if err := ethereum.StartMining(true); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/geth/misccmd.go

@@ -25,8 +25,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/ethereum/ethash"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/params"
 	"gopkg.in/urfave/cli.v1"
@@ -34,7 +34,7 @@ import (
 
 var (
 	makedagCommand = cli.Command{
-		Action:    makedag,
+		Action:    utils.MigrateFlags(makedag),
 		Name:      "makedag",
 		Usage:     "Generate ethash DAG (for testing)",
 		ArgsUsage: "<blockNum> <outputDir>",
@@ -47,7 +47,7 @@ Regular users do not need to execute it.
 `,
 	}
 	versionCommand = cli.Command{
-		Action:    version,
+		Action:    utils.MigrateFlags(version),
 		Name:      "version",
 		Usage:     "Print version numbers",
 		ArgsUsage: " ",
@@ -57,7 +57,7 @@ The output of this command is supposed to be machine-readable.
 `,
 	}
 	licenseCommand = cli.Command{
-		Action:    license,
+		Action:    utils.MigrateFlags(license),
 		Name:      "license",
 		Usage:     "Display license information",
 		ArgsUsage: " ",
@@ -87,7 +87,7 @@ func makedag(ctx *cli.Context) error {
 				utils.Fatalf("Can't find dir")
 			}
 			fmt.Println("making DAG, this could take awhile...")
-			ethash.MakeDAG(blockNum, dir)
+			ethash.MakeDataset(blockNum, dir)
 		}
 	default:
 		wrongArgs()
@@ -101,10 +101,11 @@ func version(ctx *cli.Context) error {
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
+	fmt.Println("Architecture:", runtime.GOARCH)
 	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
-	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
+	fmt.Println("Network Id:", eth.DefaultConfig.NetworkId)
 	fmt.Println("Go Version:", runtime.Version())
-	fmt.Println("OS:", runtime.GOOS)
+	fmt.Println("Operating System:", runtime.GOOS)
 	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
 	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
 	return nil

cmd/geth/monitorcmd.go

@@ -49,7 +49,7 @@ var (
 		Usage: "Refresh interval in seconds",
 	}
 	monitorCommand = cli.Command{
-		Action:    monitor,
+		Action:    utils.MigrateFlags(monitor), // keep track of migration progress
 		Name:      "monitor",
 		Usage:     "Monitor and visualize node metrics",
 		ArgsUsage: " ",

cmd/geth/usage.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"io"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
@@ -30,7 +31,7 @@ import (
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2016 The go-ethereum Authors
+   Copyright 2013-2017 The go-ethereum Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -64,19 +65,45 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "ETHEREUM",
 		Flags: []cli.Flag{
+			configFileFlag,
 			utils.DataDirFlag,
 			utils.KeyStoreDirFlag,
+			utils.NoUSBFlag,
 			utils.NetworkIdFlag,
-			utils.TestNetFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
 			utils.DevModeFlag,
+			utils.SyncModeFlag,
+			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
-			utils.FastSyncFlag,
-			utils.LightModeFlag,
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
 		},
 	},
+	{
+		Name: "ETHASH",
+		Flags: []cli.Flag{
+			utils.EthashCacheDirFlag,
+			utils.EthashCachesInMemoryFlag,
+			utils.EthashCachesOnDiskFlag,
+			utils.EthashDatasetDirFlag,
+			utils.EthashDatasetsInMemoryFlag,
+			utils.EthashDatasetsOnDiskFlag,
+		},
+	},
+	{
+		Name: "TRANSACTION POOL",
+		Flags: []cli.Flag{
+			utils.TxPoolPriceLimitFlag,
+			utils.TxPoolPriceBumpFlag,
+			utils.TxPoolAccountSlotsFlag,
+			utils.TxPoolGlobalSlotsFlag,
+			utils.TxPoolAccountQueueFlag,
+			utils.TxPoolGlobalQueueFlag,
+			utils.TxPoolLifetimeFlag,
+		},
+	},
 	{
 		Name: "PERFORMANCE TUNING",
 		Flags: []cli.Flag{
@@ -104,7 +131,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.WSApiFlag,
 			utils.WSAllowedOriginsFlag,
 			utils.IPCDisabledFlag,
-			utils.IPCApiFlag,
 			utils.IPCPathFlag,
 			utils.RPCCORSDomainFlag,
 			utils.JSpathFlag,
@@ -116,12 +142,15 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "NETWORKING",
 		Flags: []cli.Flag{
 			utils.BootnodesFlag,
+			utils.BootnodesV4Flag,
+			utils.BootnodesV5Flag,
 			utils.ListenPortFlag,
 			utils.MaxPeersFlag,
 			utils.MaxPendingPeersFlag,
 			utils.NATFlag,
 			utils.NoDiscoverFlag,
 			utils.DiscoveryV5Flag,
+			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
 		},
@@ -131,7 +160,6 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.MiningEnabledFlag,
 			utils.MinerThreadsFlag,
-			utils.AutoDAGFlag,
 			utils.EtherbaseFlag,
 			utils.TargetGasLimitFlag,
 			utils.GasPriceFlag,
@@ -141,43 +169,70 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "GAS PRICE ORACLE",
 		Flags: []cli.Flag{
-			utils.GpoMinGasPriceFlag,
-			utils.GpoMaxGasPriceFlag,
-			utils.GpoFullBlockRatioFlag,
-			utils.GpobaseStepDownFlag,
-			utils.GpobaseStepUpFlag,
-			utils.GpobaseCorrectionFactorFlag,
+			utils.GpoBlocksFlag,
+			utils.GpoPercentileFlag,
 		},
 	},
 	{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
-			utils.VMEnableJitFlag,
-			utils.VMForceJitFlag,
-			utils.VMJitCacheFlag,
 			utils.VMEnableDebugFlag,
 		},
 	},
 	{
 		Name: "LOGGING AND DEBUGGING",
 		Flags: append([]cli.Flag{
-			utils.EthStatsURLFlag,
 			utils.MetricsEnabledFlag,
 			utils.FakePoWFlag,
+			utils.NoCompactionFlag,
 		}, debug.Flags...),
 	},
+	{
+		Name: "DEPRECATED",
+		Flags: []cli.Flag{
+			utils.FastSyncFlag,
+			utils.LightModeFlag,
+		},
+	},
 	{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
 		},
 	},
-	{
-		Name: "MISCELLANEOUS",
-		Flags: []cli.Flag{
-			utils.SolcPathFlag,
-		},
-	},
+}
+
+// byCategory sorts an array of flagGroup by Name in the order
+// defined in AppHelpFlagGroups.
+type byCategory []flagGroup
+
+func (a byCategory) Len() int      { return len(a) }
+func (a byCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byCategory) Less(i, j int) bool {
+	iCat, jCat := a[i].Name, a[j].Name
+	iIdx, jIdx := len(AppHelpFlagGroups), len(AppHelpFlagGroups) // ensure non categorized flags come last
+
+	for i, group := range AppHelpFlagGroups {
+		if iCat == group.Name {
+			iIdx = i
+		}
+		if jCat == group.Name {
+			jIdx = i
+		}
+	}
+
+	return iIdx < jIdx
+}
+
+func flagCategory(flag cli.Flag) string {
+	for _, category := range AppHelpFlagGroups {
+		for _, flg := range category.Flags {
+			if flg.GetName() == flag.GetName() {
+				return category.Name
+			}
+		}
+	}
+	return "MISC"
 }
 
 func init() {
@@ -189,6 +244,7 @@ func init() {
 		App        interface{}
 		FlagGroups []flagGroup
 	}
+
 	// Override the default app help printer, but only for the global app help
 	originalHelpPrinter := cli.HelpPrinter
 	cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
@@ -218,6 +274,27 @@ func init() {
 			}
 			// Render out custom usage screen
 			originalHelpPrinter(w, tmpl, helpData{data, AppHelpFlagGroups})
+		} else if tmpl == utils.CommandHelpTemplate {
+			// Iterate over all command specific flags and categorize them
+			categorized := make(map[string][]cli.Flag)
+			for _, flag := range data.(cli.Command).Flags {
+				if _, ok := categorized[flag.String()]; !ok {
+					categorized[flagCategory(flag)] = append(categorized[flagCategory(flag)], flag)
+				}
+			}
+
+			// sort to get a stable ordering
+			sorted := make([]flagGroup, 0, len(categorized))
+			for cat, flgs := range categorized {
+				sorted = append(sorted, flagGroup{cat, flgs})
+			}
+			sort.Sort(byCategory(sorted))
+
+			// add sorted array to data and render with default printer
+			originalHelpPrinter(w, tmpl, map[string]interface{}{
+				"cmd":              data,
+				"categorizedFlags": sorted,
+			})
 		} else {
 			originalHelpPrinter(w, tmpl, data)
 		}

cmd/gethrpctest/main.go

@@ -1,162 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// gethrpctest is a command to run the external RPC tests.
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"os/signal"
-
-	"github.com/ethereum/go-ethereum/accounts/keystore"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/tests"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv2"
-)
-
-const defaultTestKey = "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"
-
-var (
-	testFile = flag.String("json", "", "Path to the .json test file to load")
-	testName = flag.String("test", "", "Name of the test from the .json file to run")
-	testKey  = flag.String("key", defaultTestKey, "Private key of a test account to inject")
-)
-
-func main() {
-	flag.Parse()
-
-	// Enable logging errors, we really do want to see those
-	glog.SetV(2)
-	glog.SetToStderr(true)
-
-	// Load the test suite to run the RPC against
-	tests, err := tests.LoadBlockTests(*testFile)
-	if err != nil {
-		log.Fatalf("Failed to load test suite: %v", err)
-	}
-	test, found := tests[*testName]
-	if !found {
-		log.Fatalf("Requested test (%s) not found within suite", *testName)
-	}
-
-	stack, err := MakeSystemNode(*testKey, test)
-	if err != nil {
-		log.Fatalf("Failed to assemble test stack: %v", err)
-	}
-	if err := stack.Start(); err != nil {
-		log.Fatalf("Failed to start test node: %v", err)
-	}
-	defer stack.Stop()
-
-	log.Println("Test node started...")
-
-	// Make sure the tests contained within the suite pass
-	if err := RunTest(stack, test); err != nil {
-		log.Fatalf("Failed to run the pre-configured test: %v", err)
-	}
-	log.Println("Initial test suite passed...")
-
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt)
-	<-quit
-}
-
-// MakeSystemNode configures a protocol stack for the RPC tests based on a given
-// keystore path and initial pre-state.
-func MakeSystemNode(privkey string, test *tests.BlockTest) (*node.Node, error) {
-	// Create a networkless protocol stack
-	stack, err := node.New(&node.Config{
-		UseLightweightKDF: true,
-		IPCPath:           node.DefaultIPCEndpoint(""),
-		HTTPHost:          node.DefaultHTTPHost,
-		HTTPPort:          node.DefaultHTTPPort,
-		HTTPModules:       []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"},
-		WSHost:            node.DefaultWSHost,
-		WSPort:            node.DefaultWSPort,
-		WSModules:         []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"},
-		NoDiscovery:       true,
-	})
-	if err != nil {
-		return nil, err
-	}
-	// Create the keystore and inject an unlocked account if requested
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
-
-	if len(privkey) > 0 {
-		key, err := crypto.HexToECDSA(privkey)
-		if err != nil {
-			return nil, err
-		}
-		a, err := ks.ImportECDSA(key, "")
-		if err != nil {
-			return nil, err
-		}
-		if err := ks.Unlock(a, ""); err != nil {
-			return nil, err
-		}
-	}
-	// Initialize and register the Ethereum protocol
-	db, _ := ethdb.NewMemDatabase()
-	if _, err := test.InsertPreState(db); err != nil {
-		return nil, err
-	}
-	ethConf := &eth.Config{
-		TestGenesisState: db,
-		TestGenesisBlock: test.Genesis,
-		ChainConfig:      &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock},
-	}
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return eth.New(ctx, ethConf) }); err != nil {
-		return nil, err
-	}
-	// Initialize and register the Whisper protocol
-	if err := stack.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(), nil }); err != nil {
-		return nil, err
-	}
-	return stack, nil
-}
-
-// RunTest executes the specified test against an already pre-configured protocol
-// stack to ensure basic checks pass before running RPC tests.
-func RunTest(stack *node.Node, test *tests.BlockTest) error {
-	var ethereum *eth.Ethereum
-	stack.Service(&ethereum)
-	blockchain := ethereum.BlockChain()
-
-	// Process the blocks and verify the imported headers
-	blocks, err := test.TryBlocksInsert(blockchain)
-	if err != nil {
-		return err
-	}
-	if err := test.ValidateImportedHeaders(blockchain, blocks); err != nil {
-		return err
-	}
-	// Retrieve the assembled state and validate it
-	stateDb, err := blockchain.State()
-	if err != nil {
-		return err
-	}
-	if err := test.ValidatePostState(stateDb); err != nil {
-		return err
-	}
-	return nil
-}

cmd/internal/browser/browser.go

@@ -0,0 +1,46 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package browser provides utilities for interacting with users' browsers.
+package browser
+
+import (
+	"os"
+	"os/exec"
+	"runtime"
+)
+
+// Commands returns a list of possible commands to use to open a url.
+func Commands() [][]string {
+	var cmds [][]string
+	if exe := os.Getenv("BROWSER"); exe != "" {
+		cmds = append(cmds, []string{exe})
+	}
+	switch runtime.GOOS {
+	case "darwin":
+		cmds = append(cmds, []string{"/usr/bin/open"})
+	case "windows":
+		cmds = append(cmds, []string{"cmd", "/c", "start"})
+	default:
+		cmds = append(cmds, []string{"xdg-open"})
+	}
+	cmds = append(cmds,
+		[]string{"chrome"},
+		[]string{"google-chrome"},
+		[]string{"chromium"},
+		[]string{"firefox"},
+	)
+	return cmds
+}
+
+// Open tries to open url in a browser and reports whether it succeeded.
+func Open(url string) bool {
+	for _, args := range Commands() {
+		cmd := exec.Command(args[0], append(args[1:], url)...)
+		if cmd.Start() == nil {
+			return true
+		}
+	}
+	return false
+}

cmd/puppeth/module.go

@@ -0,0 +1,152 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	// ErrServiceUnknown is returned when a service container doesn't exist.
+	ErrServiceUnknown = errors.New("service unknown")
+
+	// ErrServiceOffline is returned when a service container exists, but it is not
+	// running.
+	ErrServiceOffline = errors.New("service offline")
+
+	// ErrServiceUnreachable is returned when a service container is running, but
+	// seems to not respond to communication attempts.
+	ErrServiceUnreachable = errors.New("service unreachable")
+
+	// ErrNotExposed is returned if a web-service doesn't have an exposed port, nor
+	// a reverse-proxy in front of it to forward requests.
+	ErrNotExposed = errors.New("service not exposed, nor proxied")
+)
+
+// containerInfos is a heavily reduced version of the huge inspection dataset
+// returned from docker inspect, parsed into a form easily usable by puppeth.
+type containerInfos struct {
+	running bool              // Flag whether the container is running currently
+	envvars map[string]string // Collection of environmental variables set on the container
+	portmap map[string]int    // Port mapping from internal port/proto combos to host binds
+	volumes map[string]string // Volume mount points from container to host directories
+}
+
+// inspectContainer runs docker inspect against a running container
+func inspectContainer(client *sshClient, container string) (*containerInfos, error) {
+	// Check whether there's a container running for the service
+	out, err := client.Run(fmt.Sprintf("docker inspect %s", container))
+	if err != nil {
+		return nil, ErrServiceUnknown
+	}
+	// If yes, extract various configuration options
+	type inspection struct {
+		State struct {
+			Running bool
+		}
+		Mounts []struct {
+			Source      string
+			Destination string
+		}
+		Config struct {
+			Env []string
+		}
+		HostConfig struct {
+			PortBindings map[string][]map[string]string
+		}
+	}
+	var inspects []inspection
+	if err = json.Unmarshal(out, &inspects); err != nil {
+		return nil, err
+	}
+	inspect := inspects[0]
+
+	// Infos retrieved, parse the above into something meaningful
+	infos := &containerInfos{
+		running: inspect.State.Running,
+		envvars: make(map[string]string),
+		portmap: make(map[string]int),
+		volumes: make(map[string]string),
+	}
+	for _, envvar := range inspect.Config.Env {
+		if parts := strings.Split(envvar, "="); len(parts) == 2 {
+			infos.envvars[parts[0]] = parts[1]
+		}
+	}
+	for portname, details := range inspect.HostConfig.PortBindings {
+		if len(details) > 0 {
+			port, _ := strconv.Atoi(details[0]["HostPort"])
+			infos.portmap[portname] = port
+		}
+	}
+	for _, mount := range inspect.Mounts {
+		infos.volumes[mount.Destination] = mount.Source
+	}
+	return infos, err
+}
+
+// tearDown connects to a remote machine via SSH and terminates docker containers
+// running with the specified name in the specified network.
+func tearDown(client *sshClient, network string, service string, purge bool) ([]byte, error) {
+	// Tear down the running (or paused) container
+	out, err := client.Run(fmt.Sprintf("docker rm -f %s_%s_1", network, service))
+	if err != nil {
+		return out, err
+	}
+	// If requested, purge the associated docker image too
+	if purge {
+		return client.Run(fmt.Sprintf("docker rmi %s/%s", network, service))
+	}
+	return nil, nil
+}
+
+// resolve retrieves the hostname a service is running on either by returning the
+// actual server name and port, or preferably an nginx virtual host if available.
+func resolve(client *sshClient, network string, service string, port int) (string, error) {
+	// Inspect the service to get various configurations from it
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, service))
+	if err != nil {
+		return "", err
+	}
+	if !infos.running {
+		return "", ErrServiceOffline
+	}
+	// Container online, extract any environmental variables
+	if vhost := infos.envvars["VIRTUAL_HOST"]; vhost != "" {
+		return vhost, nil
+	}
+	return fmt.Sprintf("%s:%d", client.server, port), nil
+}
+
+// checkPort tries to connect to a remote host on a given
+func checkPort(host string, port int) error {
+	log.Trace("Verifying remote TCP connectivity", "server", host, "port", port)
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), time.Second)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}

cmd/puppeth/module_ethstats.go

@@ -0,0 +1,159 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ethstatsDockerfile is the Dockerfile required to build an ethstats backend
+// and associated monitoring site.
+var ethstatsDockerfile = `
+FROM mhart/alpine-node:latest
+
+RUN \
+  apk add --update git                                         && \
+  git clone --depth=1 https://github.com/karalabe/eth-netstats && \
+	apk del git && rm -rf /var/cache/apk/*                       && \
+	\
+  cd /eth-netstats && npm install && npm install -g grunt-cli && grunt
+
+WORKDIR /eth-netstats
+EXPOSE 3000
+
+RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: []};' > lib/utils/config.js
+
+CMD ["npm", "start"]
+`
+
+// ethstatsComposefile is the docker-compose.yml file required to deploy and
+// maintain an ethstats monitoring site.
+var ethstatsComposefile = `
+version: '2'
+services:
+  ethstats:
+    build: .
+    image: {{.Network}}/ethstats{{if not .VHost}}
+    ports:
+      - "{{.Port}}:3000"{{end}}
+    environment:
+      - WS_SECRET={{.Secret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}{{end}}
+    restart: always
+`
+
+// deployEthstats deploys a new ethstats container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	for i, address := range trusted {
+		trusted[i] = fmt.Sprintf("\"%s\"", address)
+	}
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"Trusted": strings.Join(trusted, ", "),
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+		"Secret":  secret,
+		"VHost":   vhost,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// ethstatsInfos is returned from an ethstats status check to allow reporting
+// various configuration parameters.
+type ethstatsInfos struct {
+	host   string
+	port   int
+	secret string
+	config string
+}
+
+// String implements the stringer interface.
+func (info *ethstatsInfos) String() string {
+	return fmt.Sprintf("host=%s, port=%d, secret=%s", info.host, info.port, info.secret)
+}
+
+// checkEthstats does a health-check against an ethstats server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkEthstats(client *sshClient, network string) (*ethstatsInfos, error) {
+	// Inspect a possible ethstats container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_ethstats_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["3000/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and configure the connection string
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	secret := infos.envvars["WS_SECRET"]
+	config := fmt.Sprintf("%s@%s", secret, host)
+	if port != 80 && port != 443 {
+		config += fmt.Sprintf(":%d", port)
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Ethstats service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &ethstatsInfos{
+		host:   host,
+		port:   port,
+		secret: secret,
+		config: config,
+	}, nil
+}

cmd/puppeth/module_faucet.go

@@ -0,0 +1,227 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// faucetDockerfile is the Dockerfile required to build an faucet container to
+// grant crypto tokens based on GitHub authentications.
+var faucetDockerfile = `
+FROM alpine:latest
+
+RUN mkdir /go
+ENV GOPATH /go
+
+RUN \
+  apk add --update git go make gcc musl-dev ca-certificates linux-headers                             && \
+	mkdir -p $GOPATH/src/github.com/ethereum                                                            && \
+	(cd $GOPATH/src/github.com/ethereum && git clone --depth=1 https://github.com/ethereum/go-ethereum) && \
+  go build -v github.com/ethereum/go-ethereum/cmd/faucet                                              && \
+  apk del git go make gcc musl-dev linux-headers                                                      && \
+  rm -rf $GOPATH && rm -rf /var/cache/apk/*
+
+ADD genesis.json /genesis.json
+ADD account.json /account.json
+ADD account.pass /account.pass
+
+EXPOSE 8080
+
+CMD [ \
+	"/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", "--ethport", "{{.EthPort}}", \
+	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",          \
+	"--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass"                       \
+	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}                                                        \
+]`
+
+// faucetComposefile is the docker-compose.yml file required to deploy and maintain
+// a crypto faucet.
+var faucetComposefile = `
+version: '2'
+services:
+  faucet:
+    build: .
+    image: {{.Network}}/faucet
+    ports:
+      - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}}
+      - "{{.ApiPort}}:8080"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.faucet
+    environment:
+      - ETH_PORT={{.EthPort}}
+      - ETH_NAME={{.EthName}}
+      - FAUCET_AMOUNT={{.FaucetAmount}}
+      - FAUCET_MINUTES={{.FaucetMinutes}}
+      - FAUCET_TIERS={{.FaucetTiers}}
+      - GITHUB_USER={{.GitHubUser}}
+      - GITHUB_TOKEN={{.GitHubToken}}
+      - CAPTCHA_TOKEN={{.CaptchaToken}}
+      - CAPTCHA_SECRET={{.CaptchaSecret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}
+      - VIRTUAL_PORT=8080{{end}}
+    restart: always
+`
+
+// deployFaucet deploys a new faucet container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployFaucet(client *sshClient, network string, bootnodes []string, config *faucetInfos) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID":     config.node.network,
+		"Bootnodes":     strings.Join(bootnodes, ","),
+		"Ethstats":      config.node.ethstats,
+		"EthPort":       config.node.portFull,
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"CaptchaToken":  config.captchaToken,
+		"CaptchaSecret": config.captchaSecret,
+		"FaucetName":    strings.Title(network),
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+		"FaucetTiers":   config.tiers,
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetComposefile)).Execute(composefile, map[string]interface{}{
+		"Network":       network,
+		"Datadir":       config.node.datadir,
+		"VHost":         config.host,
+		"ApiPort":       config.port,
+		"EthPort":       config.node.portFull,
+		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"CaptchaToken":  config.captchaToken,
+		"CaptchaSecret": config.captchaSecret,
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+		"FaucetTiers":   config.tiers,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.node.genesis)
+	files[filepath.Join(workdir, "account.json")] = []byte(config.node.keyJSON)
+	files[filepath.Join(workdir, "account.pass")] = []byte(config.node.keyPass)
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the faucet service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// faucetInfos is returned from an faucet status check to allow reporting various
+// configuration parameters.
+type faucetInfos struct {
+	node          *nodeInfos
+	host          string
+	port          int
+	amount        int
+	minutes       int
+	tiers         int
+	githubUser    string
+	githubToken   string
+	captchaToken  string
+	captchaSecret string
+}
+
+// String implements the stringer interface.
+func (info *faucetInfos) String() string {
+	return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, tiers=%d, github=%s, captcha=%v, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.tiers, info.githubUser, info.captchaToken != "", info.node.ethstats)
+}
+
+// checkFaucet does a health-check against an faucet server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
+	// Inspect a possible faucet container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_faucet_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["8080/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and the config values
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	amount, _ := strconv.Atoi(infos.envvars["FAUCET_AMOUNT"])
+	minutes, _ := strconv.Atoi(infos.envvars["FAUCET_MINUTES"])
+	tiers, _ := strconv.Atoi(infos.envvars["FAUCET_TIERS"])
+
+	// Retrieve the funding account informations
+	var out []byte
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.json", network)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.pass", network)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Faucet service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &faucetInfos{
+		node: &nodeInfos{
+			datadir:  infos.volumes["/root/.faucet"],
+			portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			ethstats: infos.envvars["ETH_NAME"],
+			keyJSON:  keyJSON,
+			keyPass:  keyPass,
+		},
+		host:          host,
+		port:          port,
+		amount:        amount,
+		minutes:       minutes,
+		tiers:         tiers,
+		githubUser:    infos.envvars["GITHUB_USER"],
+		githubToken:   infos.envvars["GITHUB_TOKEN"],
+		captchaToken:  infos.envvars["CAPTCHA_TOKEN"],
+		captchaSecret: infos.envvars["CAPTCHA_SECRET"],
+	}, nil
+}

cmd/puppeth/module_nginx.go

@@ -0,0 +1,106 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// nginxDockerfile is theis the Dockerfile required to build an nginx reverse-
+// proxy.
+var nginxDockerfile = `FROM jwilder/nginx-proxy`
+
+// nginxComposefile is the docker-compose.yml file required to deploy and maintain
+// an nginx reverse-proxy. The proxy is responsible for exposing one or more HTTP
+// services running on a single host.
+var nginxComposefile = `
+version: '2'
+services:
+  nginx:
+    build: .
+    image: {{.Network}}/nginx
+    ports:
+      - "{{.Port}}:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    restart: always
+`
+
+// deployNginx deploys a new nginx reverse-proxy container to expose one or more
+// HTTP services running on a single host. If an instance with the specified
+// network name already exists there, it will be overwritten!
+func deployNginx(client *sshClient, network string, port int) ([]byte, error) {
+	log.Info("Deploying nginx reverse-proxy", "server", client.server, "port", port)
+
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxDockerfile)).Execute(dockerfile, nil)
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nginxInfos is returned from an nginx reverse-proxy status check to allow
+// reporting various configuration parameters.
+type nginxInfos struct {
+	port int
+}
+
+// String implements the stringer interface.
+func (info *nginxInfos) String() string {
+	return fmt.Sprintf("port=%d", info.port)
+}
+
+// checkNginx does a health-check against an nginx reverse-proxy to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkNginx(client *sshClient, network string) (*nginxInfos, error) {
+	// Inspect a possible nginx container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_nginx_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Container available, assemble and return the useful infos
+	return &nginxInfos{
+		port: infos.portmap["80/tcp"],
+	}, nil
+}

cmd/puppeth/module_node.go

@@ -0,0 +1,237 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// nodeDockerfile is the Dockerfile required to run an Ethereum node.
+var nodeDockerfile = `
+FROM ethereum/client-go:alpine-develop
+
+ADD genesis.json /genesis.json
+{{if .Unlock}}
+	ADD signer.json /signer.json
+	ADD signer.pass /signer.pass
+{{end}}
+RUN \
+  echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
+	echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+
+ENTRYPOINT ["/bin/sh", "geth.sh"]
+`
+
+// nodeComposefile is the docker-compose.yml file required to deploy and maintain
+// an Ethereum node (bootnode or miner for now).
+var nodeComposefile = `
+version: '2'
+services:
+  {{.Type}}:
+    build: .
+    image: {{.Network}}/{{.Type}}
+    ports:
+      - "{{.FullPort}}:{{.FullPort}}"
+      - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
+      - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.ethereum
+    environment:
+      - FULL_PORT={{.FullPort}}/tcp
+      - LIGHT_PORT={{.LightPort}}/udp
+      - TOTAL_PEERS={{.TotalPeers}}
+      - LIGHT_PEERS={{.LightPeers}}
+      - STATS_NAME={{.Ethstats}}
+      - MINER_NAME={{.Etherbase}}
+      - GAS_TARGET={{.GasTarget}}
+      - GAS_PRICE={{.GasPrice}}
+    restart: always
+`
+
+// deployNode deploys a new Ethereum node container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos) ([]byte, error) {
+	kind := "sealnode"
+	if config.keyJSON == "" && config.etherbase == "" {
+		kind = "bootnode"
+		bootv4 = make([]string, 0)
+		bootv5 = make([]string, 0)
+	}
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	lightFlag := ""
+	if config.peersLight > 0 {
+		lightFlag = fmt.Sprintf("--lightpeers=%d --lightserv=50", config.peersLight)
+	}
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID": config.network,
+		"Port":      config.portFull,
+		"Peers":     config.peersTotal,
+		"LightFlag": lightFlag,
+		"BootV4":    strings.Join(bootv4, ","),
+		"BootV5":    strings.Join(bootv5, ","),
+		"Ethstats":  config.ethstats,
+		"Etherbase": config.etherbase,
+		"GasTarget": uint64(1000000 * config.gasTarget),
+		"GasPrice":  uint64(1000000000 * config.gasPrice),
+		"Unlock":    config.keyJSON != "",
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeComposefile)).Execute(composefile, map[string]interface{}{
+		"Type":       kind,
+		"Datadir":    config.datadir,
+		"Network":    network,
+		"FullPort":   config.portFull,
+		"TotalPeers": config.peersTotal,
+		"Light":      config.peersLight > 0,
+		"LightPort":  config.portFull + 1,
+		"LightPeers": config.peersLight,
+		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
+		"Etherbase":  config.etherbase,
+		"GasTarget":  config.gasTarget,
+		"GasPrice":   config.gasPrice,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	//genesisfile, _ := json.MarshalIndent(config.genesis, "", "  ")
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.genesis)
+
+	if config.keyJSON != "" {
+		files[filepath.Join(workdir, "signer.json")] = []byte(config.keyJSON)
+		files[filepath.Join(workdir, "signer.pass")] = []byte(config.keyPass)
+	}
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the boot or seal node service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nodeInfos is returned from a boot or seal node status check to allow reporting
+// various configuration parameters.
+type nodeInfos struct {
+	genesis    []byte
+	network    int64
+	datadir    string
+	ethstats   string
+	portFull   int
+	portLight  int
+	enodeFull  string
+	enodeLight string
+	peersTotal int
+	peersLight int
+	etherbase  string
+	keyJSON    string
+	keyPass    string
+	gasTarget  float64
+	gasPrice   float64
+}
+
+// String implements the stringer interface.
+func (info *nodeInfos) String() string {
+	discv5 := ""
+	if info.peersLight > 0 {
+		discv5 = fmt.Sprintf(", portv5=%d", info.portLight)
+	}
+	return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s, gastarget=%0.3f MGas, gasprice=%0.3f GWei",
+		info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats, info.gasTarget, info.gasPrice)
+}
+
+// checkNode does a health-check against an boot or seal node server to verify
+// whether it's running, and if yes, whether it's responsive.
+func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) {
+	kind := "bootnode"
+	if !boot {
+		kind = "sealnode"
+	}
+	// Inspect a possible bootnode container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, kind))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve a few types from the environmental variables
+	totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
+	lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
+	gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64)
+	gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64)
+
+	// Container available, retrieve its node ID and its genesis json
+	var out []byte
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 /geth --exec admin.nodeInfo.id attach", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	genesis := bytes.TrimSpace(out)
+
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.json", network, kind)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.pass", network, kind)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the devp2p is reachable
+	port := infos.portmap[infos.envvars["FULL_PORT"]]
+	if err = checkPort(client.server, port); err != nil {
+		log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
+	}
+	// Assemble and return the useful infos
+	stats := &nodeInfos{
+		genesis:    genesis,
+		datadir:    infos.volumes["/root/.ethereum"],
+		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
+		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
+		peersTotal: totalPeers,
+		peersLight: lightPeers,
+		ethstats:   infos.envvars["STATS_NAME"],
+		etherbase:  infos.envvars["MINER_NAME"],
+		keyJSON:    keyJSON,
+		keyPass:    keyPass,
+		gasTarget:  gasTarget,
+		gasPrice:   gasPrice,
+	}
+	stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
+	if stats.portLight != 0 {
+		stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
+	}
+	return stats, nil
+}

cmd/puppeth/puppeth.go

@@ -0,0 +1,55 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// puppeth is a command to assemble and maintain private networks.
+package main
+
+import (
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// main is just a boring entry point to set up the CLI app.
+func main() {
+	app := cli.NewApp()
+	app.Name = "puppeth"
+	app.Usage = "assemble and maintain private Ethereum networks"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "network",
+			Usage: "name of the network to administer",
+		},
+		cli.IntFlag{
+			Name:  "loglevel",
+			Value: 4,
+			Usage: "log level to emit to the screen",
+		},
+	}
+	app.Action = func(c *cli.Context) error {
+		// Set up the logger to print everything and the random generator
+		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
+		rand.Seed(time.Now().UnixNano())
+
+		// Start the wizard and relinquish control
+		makeWizard(c.String("network")).run()
+		return nil
+	}
+	app.Run(os.Args)
+}

cmd/puppeth/ssh.go

@@ -0,0 +1,234 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// sshClient is a small wrapper around Go's SSH client with a few utility methods
+// implemented on top.
+type sshClient struct {
+	server  string // Server name or IP without port number
+	address string // IP address of the remote server
+	pubkey  []byte // RSA public key to authenticate the server
+	client  *ssh.Client
+	logger  log.Logger
+}
+
+// dial establishes an SSH connection to a remote node using the current user and
+// the user's configured private RSA key. If that fails, password authentication
+// is fallen back to. The caller may override the login user via user@server:port.
+func dial(server string, pubkey []byte) (*sshClient, error) {
+	// Figure out a label for the server and a logger
+	label := server
+	if strings.Contains(label, ":") {
+		label = label[:strings.Index(label, ":")]
+	}
+	login := ""
+	if strings.Contains(server, "@") {
+		login = label[:strings.Index(label, "@")]
+		label = label[strings.Index(label, "@")+1:]
+		server = server[strings.Index(server, "@")+1:]
+	}
+	logger := log.New("server", label)
+	logger.Debug("Attempting to establish SSH connection")
+
+	user, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+	if login == "" {
+		login = user.Username
+	}
+	// Configure the supported authentication methods (private key and password)
+	var auths []ssh.AuthMethod
+
+	path := filepath.Join(user.HomeDir, ".ssh", "id_rsa")
+	if buf, err := ioutil.ReadFile(path); err != nil {
+		log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
+	} else {
+		key, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			log.Warn("Bad SSH key, falling back to passwords", "path", path, "err", err)
+		} else {
+			auths = append(auths, ssh.PublicKeys(key))
+		}
+	}
+	auths = append(auths, ssh.PasswordCallback(func() (string, error) {
+		fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server)
+		blob, err := terminal.ReadPassword(int(syscall.Stdin))
+
+		fmt.Println()
+		return string(blob), err
+	}))
+	// Resolve the IP address of the remote server
+	addr, err := net.LookupHost(label)
+	if err != nil {
+		return nil, err
+	}
+	if len(addr) == 0 {
+		return nil, errors.New("no IPs associated with domain")
+	}
+	// Try to dial in to the remote server
+	logger.Trace("Dialing remote SSH server", "user", login)
+	if !strings.Contains(server, ":") {
+		server += ":22"
+	}
+	keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		// If no public key is known for SSH, ask the user to confirm
+		if pubkey == nil {
+			fmt.Printf("The authenticity of host '%s (%s)' can't be established.\n", hostname, remote)
+			fmt.Printf("SSH key fingerprint is %s [MD5]\n", ssh.FingerprintLegacyMD5(key))
+			fmt.Printf("Are you sure you want to continue connecting (yes/no)? ")
+
+			text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+			switch {
+			case err != nil:
+				return err
+			case strings.TrimSpace(text) == "yes":
+				pubkey = key.Marshal()
+				return nil
+			default:
+				return fmt.Errorf("unknown auth choice: %v", text)
+			}
+		}
+		// If a public key exists for this SSH server, check that it matches
+		if bytes.Compare(pubkey, key.Marshal()) == 0 {
+			return nil
+		}
+		// We have a mismatch, forbid connecting
+		return errors.New("ssh key mismatch, readd the machine to update")
+	}
+	client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck})
+	if err != nil {
+		return nil, err
+	}
+	// Connection established, return our utility wrapper
+	c := &sshClient{
+		server:  label,
+		address: addr[0],
+		pubkey:  pubkey,
+		client:  client,
+		logger:  logger,
+	}
+	if err := c.init(); err != nil {
+		client.Close()
+		return nil, err
+	}
+	return c, nil
+}
+
+// init runs some initialization commands on the remote server to ensure it's
+// capable of acting as puppeth target.
+func (client *sshClient) init() error {
+	client.logger.Debug("Verifying if docker is available")
+	if out, err := client.Run("docker version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker configured incorrectly: %s", out)
+	}
+	client.logger.Debug("Verifying if docker-compose is available")
+	if out, err := client.Run("docker-compose version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker-compose configured incorrectly: %s", out)
+	}
+	return nil
+}
+
+// Close terminates the connection to an SSH server.
+func (client *sshClient) Close() error {
+	return client.client.Close()
+}
+
+// Run executes a command on the remote server and returns the combined output
+// along with any error status.
+func (client *sshClient) Run(cmd string) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Execute the command and return any output
+	client.logger.Trace("Running command on remote server", "cmd", cmd)
+	return session.CombinedOutput(cmd)
+}
+
+// Stream executes a command on the remote server and streams all outputs into
+// the local stdout and stderr streams.
+func (client *sshClient) Stream(cmd string) error {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	session.Stdout = os.Stdout
+	session.Stderr = os.Stderr
+
+	// Execute the command and return any output
+	client.logger.Trace("Streaming command on remote server", "cmd", cmd)
+	return session.Run(cmd)
+}
+
+// Upload copied the set of files to a remote server via SCP, creating any non-
+// existing folder in te mean time.
+func (client *sshClient) Upload(files map[string][]byte) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Create a goroutine that streams the SCP content
+	go func() {
+		out, _ := session.StdinPipe()
+		defer out.Close()
+
+		for file, content := range files {
+			client.logger.Trace("Uploading file to server", "file", file, "bytes", len(content))
+
+			fmt.Fprintln(out, "D0755", 0, filepath.Dir(file))             // Ensure the folder exists
+			fmt.Fprintln(out, "C0644", len(content), filepath.Base(file)) // Create the actual file
+			out.Write(content)                                            // Stream the data content
+			fmt.Fprint(out, "\x00")                                       // Transfer end with \x00
+			fmt.Fprintln(out, "E")                                        // Leave directory (simpler)
+		}
+	}()
+	return session.CombinedOutput("/usr/bin/scp -v -tr ./")
+}

cmd/puppeth/wizard.go

@@ -0,0 +1,281 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// config contains all the configurations needed by puppeth that should be saved
+// between sessions.
+type config struct {
+	path      string        // File containing the configuration values
+	genesis   *core.Genesis // Genesis block to cache for node deploys
+	bootFull  []string      // Bootnodes to always connect to by full nodes
+	bootLight []string      // Bootnodes to always connect to by light nodes
+	ethstats  string        // Ethstats settings to cache for node deploys
+
+	Servers map[string][]byte `json:"servers,omitempty"`
+}
+
+// servers retrieves an alphabetically sorted list of servers.
+func (c config) servers() []string {
+	servers := make([]string, 0, len(c.Servers))
+	for server := range c.Servers {
+		servers = append(servers, server)
+	}
+	sort.Strings(servers)
+
+	return servers
+}
+
+// flush dumps the contents of config to disk.
+func (c config) flush() {
+	os.MkdirAll(filepath.Dir(c.path), 0755)
+
+	out, _ := json.MarshalIndent(c, "", "  ")
+	if err := ioutil.WriteFile(c.path, out, 0644); err != nil {
+		log.Warn("Failed to save puppeth configs", "file", c.path, "err", err)
+	}
+}
+
+type wizard struct {
+	network string // Network name to manage
+	conf    config // Configurations from previous runs
+
+	servers  map[string]*sshClient // SSH connections to servers to administer
+	services map[string][]string   // Ethereum services known to be running on servers
+
+	in *bufio.Reader // Wrapper around stdin to allow reading user input
+}
+
+// read reads a single line from stdin, trimming if from spaces.
+func (w *wizard) read() string {
+	fmt.Printf("> ")
+	text, err := w.in.ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+	return strings.TrimSpace(text)
+}
+
+// readString reads a single line from stdin, trimming if from spaces, enforcing
+// non-emptyness.
+func (w *wizard) readString() string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+	}
+}
+
+// readDefaultString reads a single line from stdin, trimming if from spaces. If
+// an empty line is entered, the default value is returned.
+func (w *wizard) readDefaultString(def string) string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+		return def
+	}
+}
+
+// readInt reads a single line from stdin, trimming if from spaces, enforcing it
+// to parse into an integer.
+func (w *wizard) readInt() int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			continue
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readDefaultInt reads a single line from stdin, trimming if from spaces, enforcing
+// it to parse into an integer. If an empty line is entered, the default value is
+// returned.
+func (w *wizard) readDefaultInt(def int) int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readFloat reads a single line from stdin, trimming if from spaces, enforcing it
+// to parse into a float.
+func (w *wizard) readFloat() float64 {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			continue
+		}
+		val, err := strconv.ParseFloat(strings.TrimSpace(text), 64)
+		if err != nil {
+			log.Error("Invalid input, expected float", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readDefaultFloat reads a single line from stdin, trimming if from spaces, enforcing
+// it to parse into a float. If an empty line is entered, the default value is returned.
+func (w *wizard) readDefaultFloat(def float64) float64 {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, err := strconv.ParseFloat(strings.TrimSpace(text), 64)
+		if err != nil {
+			log.Error("Invalid input, expected float", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readPassword reads a single line from stdin, trimming it from the trailing new
+// line and returns it. The input will not be echoed.
+func (w *wizard) readPassword() string {
+	for {
+		fmt.Printf("> ")
+		text, err := terminal.ReadPassword(int(syscall.Stdin))
+		if err != nil {
+			log.Crit("Failed to read password", "err", err)
+		}
+		fmt.Println()
+		return string(text)
+	}
+}
+
+// readAddress reads a single line from stdin, trimming if from spaces and converts
+// it to an Ethereum address.
+func (w *wizard) readAddress() *common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return nil
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		address := common.BigToAddress(bigaddr)
+		return &address
+	}
+}
+
+// readDefaultAddress reads a single line from stdin, trimming if from spaces and
+// converts it to an Ethereum address. If an empty line is entered, the default
+// value is returned.
+func (w *wizard) readDefaultAddress(def common.Address) common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		return common.BigToAddress(bigaddr)
+	}
+}
+
+// readJSON reads a raw JSON message and returns it.
+func (w *wizard) readJSON() string {
+	var blob json.RawMessage
+
+	for {
+		fmt.Printf("> ")
+		if err := json.NewDecoder(w.in).Decode(&blob); err != nil {
+			log.Error("Invalid JSON, please try again", "err", err)
+			continue
+		}
+		return string(blob)
+	}
+}

cmd/puppeth/wizard_dashboard.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// deployDashboard queries the user for various input on deploying a web-service
+// dashboard, after which is pushes the container.
+func (w *wizard) deployDashboard() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active dashboard configurations from the server
+	infos, err := checkDashboard(client, w.network)
+	if err != nil {
+		infos = &dashboardInfos{
+			port: 80,
+			host: client.server,
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the dashboard listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy the dashboard on
+	infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host)
+	if err != nil {
+		log.Error("Failed to decide on dashboard host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out which services are available
+	available := make(map[string][]string)
+	for server, services := range w.services {
+		for _, service := range services {
+			available[service] = append(available[service], server)
+		}
+	}
+	listing := make(map[string]string)
+	for _, service := range []string{"ethstats", "explorer", "wallet", "faucet"} {
+		// Gather all the locally hosted pages of this type
+		var pages []string
+		for _, server := range available[service] {
+			client := w.servers[server]
+			if client == nil {
+				continue
+			}
+			// If there's a service running on the machine, retrieve it's port number
+			var port int
+			switch service {
+			case "ethstats":
+				if infos, err := checkEthstats(client, w.network); err == nil {
+					port = infos.port
+				}
+			case "faucet":
+				if infos, err := checkFaucet(client, w.network); err == nil {
+					port = infos.port
+				}
+			}
+			if page, err := resolve(client, w.network, service, port); err == nil && page != "" {
+				pages = append(pages, page)
+			}
+		}
+		// Promt the user to chose one, enter manually or simply not list this service
+		defLabel, defChoice := "don't list", len(pages)+2
+		if len(pages) > 0 {
+			defLabel, defChoice = pages[0], 1
+		}
+		fmt.Println()
+		fmt.Printf("Which %s service to list? (default = %s)\n", service, defLabel)
+		for i, page := range pages {
+			fmt.Printf(" %d. %s\n", i+1, page)
+		}
+		fmt.Printf(" %d. List external %s service\n", len(pages)+1, service)
+		fmt.Printf(" %d. Don't list any %s service\n", len(pages)+2, service)
+
+		choice := w.readDefaultInt(defChoice)
+		if choice < 0 || choice > len(pages)+2 {
+			log.Error("Invalid listing choice, aborting")
+			return
+		}
+		switch {
+		case choice <= len(pages):
+			listing[service] = pages[choice-1]
+		case choice == len(pages)+1:
+			fmt.Println()
+			fmt.Printf("Which address is the external %s service at?\n", service)
+			listing[service] = w.readString()
+		default:
+			// No service hosting for this
+		}
+	}
+	// If we have ethstats running, ask whether to make the secret public or not
+	var ethstats bool
+	if w.conf.ethstats != "" {
+		fmt.Println()
+		fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)")
+		ethstats = w.readDefaultString("y") == "y"
+	}
+	// Try to deploy the dashboard container on the host
+	if out, err := deployDashboard(client, w.network, infos.port, infos.host, listing, &w.conf, ethstats); err != nil {
+		log.Error("Failed to deploy dashboard container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_ethstats.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// deployEthstats queries the user for various input on deploying an ethstats
+// monitoring server, after which it executes it.
+func (w *wizard) deployEthstats() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkEthstats(client, w.network)
+	if err != nil {
+		infos = &ethstatsInfos{
+			port:   80,
+			host:   client.server,
+			secret: "",
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should ethstats listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on ethstats host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the secret and boot ethstats
+	fmt.Println()
+	if infos.secret == "" {
+		fmt.Printf("What should be the secret password for the API? (must not be empty)\n")
+		infos.secret = w.readString()
+	} else {
+		fmt.Printf("What should be the secret password for the API? (default = %s)\n", infos.secret)
+		infos.secret = w.readDefaultString(infos.secret)
+	}
+	// Try to deploy the ethstats server on the host
+	trusted := make([]string, 0, len(w.servers))
+	for _, client := range w.servers {
+		if client != nil {
+			trusted = append(trusted, client.address)
+		}
+	}
+	if out, err := deployEthstats(client, w.network, infos.port, infos.secret, infos.host, trusted); err != nil {
+		log.Error("Failed to deploy ethstats container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_faucet.go

@@ -0,0 +1,203 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// deployFaucet queries the user for various input on deploying a faucet, after
+// which it executes it.
+func (w *wizard) deployFaucet() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active faucet configurations from the server
+	infos, err := checkFaucet(client, w.network)
+	if err != nil {
+		infos = &faucetInfos{
+			node:    &nodeInfos{portFull: 30303, peersTotal: 25},
+			port:    80,
+			host:    client.server,
+			amount:  1,
+			minutes: 1440,
+			tiers:   3,
+		}
+	}
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.node.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the faucet listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on faucet host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the funcing amount per perdion configurations
+	fmt.Println()
+	fmt.Printf("How many Ethers to release per request? (default = %d)\n", infos.amount)
+	infos.amount = w.readDefaultInt(infos.amount)
+
+	fmt.Println()
+	fmt.Printf("How many minutes to enforce between requests? (default = %d)\n", infos.minutes)
+	infos.minutes = w.readDefaultInt(infos.minutes)
+
+	fmt.Println()
+	fmt.Printf("How many funding tiers to feature (x2.5 amounts, x3 timeout)? (default = %d)\n", infos.tiers)
+	infos.tiers = w.readDefaultInt(infos.tiers)
+	if infos.tiers == 0 {
+		log.Error("At least one funding tier must be set")
+		return
+	}
+	// Accessing GitHub gists requires API authorization, retrieve it
+	if infos.githubUser != "" {
+		fmt.Println()
+		fmt.Printf("Reuse previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser)
+		if w.readDefaultString("y") != "y" {
+			infos.githubUser, infos.githubToken = "", ""
+		}
+	}
+	if infos.githubUser == "" {
+		// No previous authorization (or new one requested)
+		fmt.Println()
+		fmt.Println("Which GitHub user to verify Gists through?")
+		infos.githubUser = w.readString()
+
+		fmt.Println()
+		fmt.Println("What is the GitHub personal access token of the user? (won't be echoed)")
+		infos.githubToken = w.readPassword()
+
+		// Do a sanity check query against github to ensure it's valid
+		req, _ := http.NewRequest("GET", "https://api.github.com/user", nil)
+		req.SetBasicAuth(infos.githubUser, infos.githubToken)
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Error("Failed to verify GitHub authentication", "err", err)
+			return
+		}
+		defer res.Body.Close()
+
+		var msg struct {
+			Login   string `json:"login"`
+			Message string `json:"message"`
+		}
+		if err = json.NewDecoder(res.Body).Decode(&msg); err != nil {
+			log.Error("Failed to decode authorization response", "err", err)
+			return
+		}
+		if msg.Login != infos.githubUser {
+			log.Error("GitHub authorization failed", "user", infos.githubUser, "message", msg.Message)
+			return
+		}
+	}
+	// Accessing the reCaptcha service requires API authorizations, request it
+	if infos.captchaToken != "" {
+		fmt.Println()
+		fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)")
+		if w.readDefaultString("y") != "y" {
+			infos.captchaToken, infos.captchaSecret = "", ""
+		}
+	}
+	if infos.captchaToken == "" {
+		// No previous authorization (or old one discarded)
+		fmt.Println()
+		fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)")
+		if w.readDefaultString("n") == "y" {
+			// Captcha protection explicitly requested, read the site and secret keys
+			fmt.Println()
+			fmt.Printf("What is the reCaptcha site key to authenticate human users?\n")
+			infos.captchaToken = w.readString()
+
+			fmt.Println()
+			fmt.Printf("What is the reCaptcha secret key to verify authentications? (won't be echoed)\n")
+			infos.captchaSecret = w.readPassword()
+		}
+	}
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.node.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.node.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.node.datadir)
+		infos.node.datadir = w.readDefaultString(infos.node.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
+	infos.node.portFull = w.readDefaultInt(infos.node.portFull)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.node.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.node.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.node.ethstats)
+		infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats
+	}
+	// Load up the credential needed to release funds
+	if infos.node.keyJSON != "" {
+		var key keystore.Key
+		if err := json.Unmarshal([]byte(infos.node.keyJSON), &key); err != nil {
+			infos.node.keyJSON, infos.node.keyPass = "", ""
+		} else {
+			fmt.Println()
+			fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex())
+			if w.readDefaultString("y") != "y" {
+				infos.node.keyJSON, infos.node.keyPass = "", ""
+			}
+		}
+	}
+	if infos.node.keyJSON == "" {
+		fmt.Println()
+		fmt.Println("Please paste the faucet's funding account key JSON:")
+		infos.node.keyJSON = w.readJSON()
+
+		fmt.Println()
+		fmt.Println("What's the unlock password for the account? (won't be echoed)")
+		infos.node.keyPass = w.readPassword()
+
+		if _, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil {
+			log.Error("Failed to decrypt key with given passphrase")
+			return
+		}
+	}
+	// Try to deploy the faucet server on the host
+	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos); err != nil {
+		log.Error("Failed to deploy faucet container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_genesis.go

@@ -0,0 +1,136 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// makeGenesis creates a new genesis struct based on some user input.
+func (w *wizard) makeGenesis() {
+	// Construct a default genesis block
+	genesis := &core.Genesis{
+		Timestamp:  uint64(time.Now().Unix()),
+		GasLimit:   4700000,
+		Difficulty: big.NewInt(1048576),
+		Alloc:      make(core.GenesisAlloc),
+		Config: &params.ChainConfig{
+			HomesteadBlock: big.NewInt(1),
+			EIP150Block:    big.NewInt(2),
+			EIP155Block:    big.NewInt(3),
+			EIP158Block:    big.NewInt(3),
+		},
+	}
+	// Figure out which consensus engine to choose
+	fmt.Println()
+	fmt.Println("Which consensus engine to use? (default = clique)")
+	fmt.Println(" 1. Ethash - proof-of-work")
+	fmt.Println(" 2. Clique - proof-of-authority")
+
+	choice := w.read()
+	switch {
+	case choice == "1":
+		// In case of ethash, we're pretty much done
+		genesis.Config.Ethash = new(params.EthashConfig)
+		genesis.ExtraData = make([]byte, 32)
+
+	case choice == "" || choice == "2":
+		// In the case of clique, configure the consensus parameters
+		genesis.Difficulty = big.NewInt(1)
+		genesis.Config.Clique = &params.CliqueConfig{
+			Period: 15,
+			Epoch:  30000,
+		}
+		fmt.Println()
+		fmt.Println("How many seconds should blocks take? (default = 15)")
+		genesis.Config.Clique.Period = uint64(w.readDefaultInt(15))
+
+		// We also need the initial list of signers
+		fmt.Println()
+		fmt.Println("Which accounts are allowed to seal? (mandatory at least one)")
+
+		var signers []common.Address
+		for {
+			if address := w.readAddress(); address != nil {
+				signers = append(signers, *address)
+				continue
+			}
+			if len(signers) > 0 {
+				break
+			}
+		}
+		// Sort the signers and embed into the extra-data section
+		for i := 0; i < len(signers); i++ {
+			for j := i + 1; j < len(signers); j++ {
+				if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
+					signers[i], signers[j] = signers[j], signers[i]
+				}
+			}
+		}
+		genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65)
+		for i, signer := range signers {
+			copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:])
+		}
+
+	default:
+		log.Crit("Invalid consensus engine choice", "choice", choice)
+	}
+	// Consensus all set, just ask for initial funds and go
+	fmt.Println()
+	fmt.Println("Which accounts should be pre-funded? (advisable at least one)")
+	for {
+		// Read the address of the account to fund
+		if address := w.readAddress(); address != nil {
+			genesis.Alloc[*address] = core.GenesisAccount{
+				Balance: new(big.Int).Lsh(big.NewInt(1), 256-7), // 2^256 / 128 (allow many pre-funds without balance overflows)
+			}
+			continue
+		}
+		break
+	}
+	// Add a batch of precompile balances to avoid them getting deleted
+	for i := int64(0); i < 256; i++ {
+		genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
+	}
+	fmt.Println()
+
+	// Query the user for some custom extras
+	fmt.Println()
+	fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)")
+	genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536))))
+
+	fmt.Println()
+	fmt.Println("Anything fun to embed into the genesis block? (max 32 bytes)")
+
+	extra := w.read()
+	if len(extra) > 32 {
+		extra = extra[:32]
+	}
+	genesis.ExtraData = append([]byte(extra), genesis.ExtraData[len(extra):]...)
+
+	// All done, store the genesis and flush to disk
+	w.conf.genesis = genesis
+}

cmd/puppeth/wizard_intro.go

@@ -0,0 +1,156 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// makeWizard creates and returns a new puppeth wizard.
+func makeWizard(network string) *wizard {
+	return &wizard{
+		network: network,
+		conf: config{
+			Servers: make(map[string][]byte),
+		},
+		servers:  make(map[string]*sshClient),
+		services: make(map[string][]string),
+		in:       bufio.NewReader(os.Stdin),
+	}
+}
+
+// run displays some useful infos to the user, starting on the journey of
+// setting up a new or managing an existing Ethereum private network.
+func (w *wizard) run() {
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println("| Welcome to puppeth, your Ethereum private network manager |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| This tool lets you create a new Ethereum network down to  |")
+	fmt.Println("| the genesis block, bootnodes, miners and ethstats servers |")
+	fmt.Println("| without the hassle that it would normally entail.         |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| Puppeth uses SSH to dial in to remote servers, and builds |")
+	fmt.Println("| its network components out of Docker containers using the |")
+	fmt.Println("| docker-compose toolset.                                   |")
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println()
+
+	// Make sure we have a good network name to work with	fmt.Println()
+	if w.network == "" {
+		fmt.Println("Please specify a network name to administer (no spaces, please)")
+		for {
+			w.network = w.readString()
+			if !strings.Contains(w.network, " ") {
+				fmt.Printf("Sweet, you can set this via --network=%s next time!\n\n", w.network)
+				break
+			}
+			log.Error("I also like to live dangerously, still no spaces")
+		}
+	}
+	log.Info("Administering Ethereum network", "name", w.network)
+
+	// Load initial configurations and connect to all live servers
+	w.conf.path = filepath.Join(os.Getenv("HOME"), ".puppeth", w.network)
+
+	blob, err := ioutil.ReadFile(w.conf.path)
+	if err != nil {
+		log.Warn("No previous configurations found", "path", w.conf.path)
+	} else if err := json.Unmarshal(blob, &w.conf); err != nil {
+		log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err)
+	} else {
+		for server, pubkey := range w.conf.Servers {
+			log.Info("Dialing previously configured server", "server", server)
+			client, err := dial(server, pubkey)
+			if err != nil {
+				log.Error("Previous server unreachable", "server", server, "err", err)
+			}
+			w.servers[server] = client
+		}
+		w.networkStats(false)
+	}
+	// Basics done, loop ad infinitum about what to do
+	for {
+		fmt.Println()
+		fmt.Println("What would you like to do? (default = stats)")
+		fmt.Println(" 1. Show network stats")
+		if w.conf.genesis == nil {
+			fmt.Println(" 2. Configure new genesis")
+		} else {
+			fmt.Println(" 2. Save existing genesis")
+		}
+		if len(w.servers) == 0 {
+			fmt.Println(" 3. Track new remote server")
+		} else {
+			fmt.Println(" 3. Manage tracked machines")
+		}
+		if len(w.services) == 0 {
+			fmt.Println(" 4. Deploy network components")
+		} else {
+			fmt.Println(" 4. Manage network components")
+		}
+		//fmt.Println(" 5. ProTips for common usecases")
+
+		choice := w.read()
+		switch {
+		case choice == "" || choice == "1":
+			w.networkStats(false)
+
+		case choice == "2":
+			// If we don't have a genesis, make one
+			if w.conf.genesis == nil {
+				w.makeGenesis()
+			} else {
+				// Otherwise just save whatever we currently have
+				fmt.Println()
+				fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
+				out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
+				if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
+					log.Error("Failed to save genesis file", "err", err)
+				}
+				log.Info("Exported existing genesis block")
+			}
+		case choice == "3":
+			if len(w.servers) == 0 {
+				if w.makeServer() != "" {
+					w.networkStats(false)
+				}
+			} else {
+				w.manageServers()
+			}
+		case choice == "4":
+			if len(w.services) == 0 {
+				w.deployComponent()
+			} else {
+				w.manageComponents()
+			}
+
+		case choice == "5":
+			w.networkStats(true)
+
+		default:
+			log.Error("That's not something I can do")
+		}
+	}
+}

cmd/puppeth/wizard_netstats.go

@@ -0,0 +1,235 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/olekukonko/tablewriter"
+)
+
+// networkStats verifies the status of network components and generates a protip
+// configuration set to give users hints on how to do various tasks.
+func (w *wizard) networkStats(tips bool) {
+	if len(w.servers) == 0 {
+		log.Error("No remote machines to gather stats from")
+		return
+	}
+	protips := new(protips)
+
+	// Iterate over all the specified hosts and check their status
+	stats := tablewriter.NewWriter(os.Stdout)
+	stats.SetHeader([]string{"Server", "IP", "Status", "Service", "Details"})
+	stats.SetColWidth(100)
+
+	for server, pubkey := range w.conf.Servers {
+		client := w.servers[server]
+		logger := log.New("server", server)
+		logger.Info("Starting remote server health-check")
+
+		// If the server is not connected, try to connect again
+		if client == nil {
+			conn, err := dial(server, pubkey)
+			if err != nil {
+				logger.Error("Failed to establish remote connection", "err", err)
+				stats.Append([]string{server, "", err.Error(), "", ""})
+				continue
+			}
+			client = conn
+		}
+		// Client connected one way or another, run health-checks
+		services := make(map[string]string)
+		logger.Debug("Checking for nginx availability")
+		if infos, err := checkNginx(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["nginx"] = err.Error()
+			}
+		} else {
+			services["nginx"] = infos.String()
+		}
+		logger.Debug("Checking for ethstats availability")
+		if infos, err := checkEthstats(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["ethstats"] = err.Error()
+			}
+		} else {
+			services["ethstats"] = infos.String()
+			protips.ethstats = infos.config
+		}
+		logger.Debug("Checking for bootnode availability")
+		if infos, err := checkNode(client, w.network, true); err != nil {
+			if err != ErrServiceUnknown {
+				services["bootnode"] = err.Error()
+			}
+		} else {
+			services["bootnode"] = infos.String()
+
+			protips.genesis = string(infos.genesis)
+			protips.bootFull = append(protips.bootFull, infos.enodeFull)
+			if infos.enodeLight != "" {
+				protips.bootLight = append(protips.bootLight, infos.enodeLight)
+			}
+		}
+		logger.Debug("Checking for sealnode availability")
+		if infos, err := checkNode(client, w.network, false); err != nil {
+			if err != ErrServiceUnknown {
+				services["sealnode"] = err.Error()
+			}
+		} else {
+			services["sealnode"] = infos.String()
+			protips.genesis = string(infos.genesis)
+		}
+		logger.Debug("Checking for faucet availability")
+		if infos, err := checkFaucet(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["faucet"] = err.Error()
+			}
+		} else {
+			services["faucet"] = infos.String()
+		}
+		logger.Debug("Checking for dashboard availability")
+		if infos, err := checkDashboard(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["dashboard"] = err.Error()
+			}
+		} else {
+			services["dashboard"] = infos.String()
+		}
+		// All status checks complete, report and check next server
+		delete(w.services, server)
+		for service := range services {
+			w.services[server] = append(w.services[server], service)
+		}
+		server, address := client.server, client.address
+		for service, status := range services {
+			stats.Append([]string{server, address, "online", service, status})
+			server, address = "", ""
+		}
+		if len(services) == 0 {
+			stats.Append([]string{server, address, "online", "", ""})
+		}
+	}
+	// If a genesis block was found, load it into our configs
+	if protips.genesis != "" {
+		genesis := new(core.Genesis)
+		if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
+			log.Error("Failed to parse remote genesis", "err", err)
+		} else {
+			w.conf.genesis = genesis
+			protips.network = genesis.Config.ChainId.Int64()
+		}
+	}
+	if protips.ethstats != "" {
+		w.conf.ethstats = protips.ethstats
+	}
+	w.conf.bootFull = protips.bootFull
+	w.conf.bootLight = protips.bootLight
+
+	// Print any collected stats and return
+	if !tips {
+		stats.Render()
+	} else {
+		protips.print(w.network)
+	}
+}
+
+// protips contains a collection of network infos to report pro-tips
+// based on.
+type protips struct {
+	genesis   string
+	network   int64
+	bootFull  []string
+	bootLight []string
+	ethstats  string
+}
+
+// print analyzes the network information available and prints a collection of
+// pro tips for the user's consideration.
+func (p *protips) print(network string) {
+	// If a known genesis block is available, display it and prepend an init command
+	fullinit, lightinit := "", ""
+	if p.genesis != "" {
+		fullinit = fmt.Sprintf("geth --datadir=$HOME/.%s init %s.json && ", network, network)
+		lightinit = fmt.Sprintf("geth --datadir=$HOME/.%s --light init %s.json && ", network, network)
+	}
+	// If an ethstats server is available, add the ethstats flag
+	statsflag := ""
+	if p.ethstats != "" {
+		if strings.Contains(p.ethstats, " ") {
+			statsflag = fmt.Sprintf(` --ethstats="yournode:%s"`, p.ethstats)
+		} else {
+			statsflag = fmt.Sprintf(` --ethstats=yournode:%s`, p.ethstats)
+		}
+	}
+	// If bootnodes have been specified, add the bootnode flag
+	bootflagFull := ""
+	if len(p.bootFull) > 0 {
+		bootflagFull = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootFull, ","))
+	}
+	bootflagLight := ""
+	if len(p.bootLight) > 0 {
+		bootflagLight = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootLight, ","))
+	}
+	// Assemble all the known pro-tips
+	var tasks, tips []string
+
+	tasks = append(tasks, "Run an archive node with historical data")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=1024%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a full node with recent data only")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=512 --fast%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a light node with on demand retrievals")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	tasks = append(tasks, "Run an embedded node with constrained memory")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=32 --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	// If the tips are short, display in a table
+	short := true
+	for _, tip := range tips {
+		if len(tip) > 100 {
+			short = false
+			break
+		}
+	}
+	fmt.Println()
+	if short {
+		howto := tablewriter.NewWriter(os.Stdout)
+		howto.SetHeader([]string{"Fun tasks for you", "Tips on how to"})
+		howto.SetColWidth(100)
+
+		for i := 0; i < len(tasks); i++ {
+			howto.Append([]string{tasks[i], tips[i]})
+		}
+		howto.Render()
+		return
+	}
+	// Meh, tips got ugly, split into many lines
+	for i := 0; i < len(tasks); i++ {
+		fmt.Println(tasks[i])
+		fmt.Println(strings.Repeat("-", len(tasks[i])))
+		fmt.Println(tips[i])
+		fmt.Println()
+		fmt.Println()
+	}
+}

cmd/puppeth/wizard_network.go

@@ -0,0 +1,198 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// manageServers displays a list of servers the user can disconnect from, and an
+// option to connect to new servers.
+func (w *wizard) manageServers() {
+	// List all the servers we can disconnect, along with an entry to connect a new one
+	fmt.Println()
+
+	servers := w.conf.servers()
+	for i, server := range servers {
+		fmt.Printf(" %d. Disconnect %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return
+	}
+	// If the user selected an existing server, drop it
+	if choice <= len(w.conf.Servers) {
+		server := servers[choice-1]
+		client := w.servers[server]
+
+		delete(w.servers, server)
+		if client != nil {
+			client.Close()
+		}
+		delete(w.conf.Servers, server)
+		w.conf.flush()
+
+		log.Info("Disconnected existing server", "server", server)
+		w.networkStats(false)
+		return
+	}
+	// If the user requested connecting a new server, do it
+	if w.makeServer() != "" {
+		w.networkStats(false)
+	}
+}
+
+// makeServer reads a single line from stdin and interprets it as a hostname to
+// connect to. It tries to establish a new SSH session and also executing some
+// baseline validations.
+//
+// If connection succeeds, the server is added to the wizards configs!
+func (w *wizard) makeServer() string {
+	fmt.Println()
+	fmt.Println("Please enter remote server's address:")
+
+	for {
+		// Read and fial the server to ensure docker is present
+		input := w.readString()
+
+		client, err := dial(input, nil)
+		if err != nil {
+			log.Error("Server not ready for puppeth", "err", err)
+			return ""
+		}
+		// All checks passed, start tracking the server
+		w.servers[input] = client
+		w.conf.Servers[input] = client.pubkey
+		w.conf.flush()
+
+		return input
+	}
+}
+
+// selectServer lists the user all the currnetly known servers to choose from,
+// also granting the option to add a new one.
+func (w *wizard) selectServer() string {
+	// List the available server to the user and wait for a choice
+	fmt.Println()
+	fmt.Println("Which server do you want to interact with?")
+
+	servers := w.conf.servers()
+	for i, server := range servers {
+		fmt.Printf(" %d. %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return ""
+	}
+	// If the user requested connecting to a new server, go for it
+	if choice <= len(w.conf.Servers) {
+		return servers[choice-1]
+	}
+	return w.makeServer()
+}
+
+// manageComponents displays a list of network components the user can tear down
+// and an option
+func (w *wizard) manageComponents() {
+	// List all the componens we can tear down, along with an entry to deploy a new one
+	fmt.Println()
+
+	var serviceHosts, serviceNames []string
+	for server, services := range w.services {
+		for _, service := range services {
+			serviceHosts = append(serviceHosts, server)
+			serviceNames = append(serviceNames, service)
+
+			fmt.Printf(" %d. Tear down %s on %s\n", len(serviceHosts), strings.Title(service), server)
+		}
+	}
+	fmt.Printf(" %d. Deploy new network component\n", len(serviceHosts)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(serviceHosts)+1 {
+		log.Error("Invalid component choice, aborting")
+		return
+	}
+	// If the user selected an existing service, destroy it
+	if choice <= len(serviceHosts) {
+		// Figure out the service to destroy and execute it
+		service := serviceNames[choice-1]
+		server := serviceHosts[choice-1]
+		client := w.servers[server]
+
+		if out, err := tearDown(client, w.network, service, true); err != nil {
+			log.Error("Failed to tear down component", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return
+		}
+		// Clean up any references to it from out state
+		services := w.services[server]
+		for i, name := range services {
+			if name == service {
+				w.services[server] = append(services[:i], services[i+1:]...)
+				if len(w.services[server]) == 0 {
+					delete(w.services, server)
+				}
+			}
+		}
+		log.Info("Torn down existing component", "server", server, "service", service)
+		return
+	}
+	// If the user requested deploying a new component, do it
+	w.deployComponent()
+}
+
+// deployComponent displays a list of network components the user can deploy and
+// guides through the process.
+func (w *wizard) deployComponent() {
+	// Print all the things we can deploy and wait or user choice
+	fmt.Println()
+	fmt.Println("What would you like to deploy? (recommended order)")
+	fmt.Println(" 1. Ethstats  - Network monitoring tool")
+	fmt.Println(" 2. Bootnode  - Entry point of the network")
+	fmt.Println(" 3. Sealer    - Full node minting new blocks")
+	fmt.Println(" 4. Wallet    - Browser wallet for quick sends (todo)")
+	fmt.Println(" 5. Faucet    - Crypto faucet to give away funds")
+	fmt.Println(" 6. Dashboard - Website listing above web-services")
+
+	switch w.read() {
+	case "1":
+		w.deployEthstats()
+	case "2":
+		w.deployNode(true)
+	case "3":
+		w.deployNode(false)
+	case "4":
+	case "5":
+		w.deployFaucet()
+	case "6":
+		w.deployDashboard()
+	default:
+		log.Error("That's not something I can do")
+	}
+}

cmd/puppeth/wizard_nginx.go

@@ -0,0 +1,58 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ensureVirtualHost checks whether a reverse-proxy is running on the specified
+// host machine, and if yes requests a virtual host from the user to host a
+// specific web service on. If no proxy exists, the method will offer to deploy
+// one.
+//
+// If the user elects not to use a reverse proxy, an empty hostname is returned!
+func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (string, error) {
+	if proxy, _ := checkNginx(client, w.network); proxy != nil {
+		// Reverse proxy is running, if ports match, we need a virtual host
+		if proxy.port == port {
+			fmt.Println()
+			fmt.Printf("Shared port, which domain to assign? (default = %s)\n", def)
+			return w.readDefaultString(def), nil
+		}
+	}
+	// Reverse proxy is not running, offer to deploy a new one
+	fmt.Println()
+	fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)")
+	if w.readDefaultString("y") == "y" {
+		if out, err := deployNginx(client, w.network, port); err != nil {
+			log.Error("Failed to deploy reverse-proxy", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return "", err
+		}
+		// Reverse proxy deployed, ask again for the virtual-host
+		fmt.Println()
+		fmt.Printf("Proxy deployed, which domain to assign? (default = %s)\n", def)
+		return w.readDefaultString(def), nil
+	}
+	// Reverse proxy not requested, deploy as a standalone service
+	return "", nil
+}

cmd/puppeth/wizard_node.go

@@ -0,0 +1,160 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// deployNode creates a new node configuration based on some user input.
+func (w *wizard) deployNode(boot bool) {
+	// Do some sanity check before the user wastes time on input
+	if w.conf.genesis == nil {
+		log.Error("No genesis block configured")
+		return
+	}
+	if w.conf.ethstats == "" {
+		log.Error("No ethstats server configured")
+		return
+	}
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkNode(client, w.network, boot)
+	if err != nil {
+		if boot {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
+		} else {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
+		}
+	}
+	infos.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		infos.datadir = w.readDefaultString(infos.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
+	infos.portFull = w.readDefaultInt(infos.portFull)
+
+	// Figure out how many peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many peers to allow connecting? (default = %d)\n", infos.peersTotal)
+	infos.peersTotal = w.readDefaultInt(infos.peersTotal)
+
+	// Figure out how many light peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many light peers to allow connecting? (default = %d)\n", infos.peersLight)
+	infos.peersLight = w.readDefaultInt(infos.peersLight)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.ethstats)
+		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+	}
+	// If the node is a miner/signer, load up needed credentials
+	if !boot {
+		if w.conf.genesis.Config.Ethash != nil {
+			// Ethash based miners only need an etherbase to mine against
+			fmt.Println()
+			if infos.etherbase == "" {
+				fmt.Printf("What address should the miner user?\n")
+				for {
+					if address := w.readAddress(); address != nil {
+						infos.etherbase = address.Hex()
+						break
+					}
+				}
+			} else {
+				fmt.Printf("What address should the miner user? (default = %s)\n", infos.etherbase)
+				infos.etherbase = w.readDefaultAddress(common.HexToAddress(infos.etherbase)).Hex()
+			}
+		} else if w.conf.genesis.Config.Clique != nil {
+			// If a previous signer was already set, offer to reuse it
+			if infos.keyJSON != "" {
+				if key, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
+					infos.keyJSON, infos.keyPass = "", ""
+				} else {
+					fmt.Println()
+					fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex())
+					if w.readDefaultString("y") != "y" {
+						infos.keyJSON, infos.keyPass = "", ""
+					}
+				}
+			}
+			// Clique based signers need a keyfile and unlock password, ask if unavailable
+			if infos.keyJSON == "" {
+				fmt.Println()
+				fmt.Println("Please paste the signer's key JSON:")
+				infos.keyJSON = w.readJSON()
+
+				fmt.Println()
+				fmt.Println("What's the unlock password for the account? (won't be echoed)")
+				infos.keyPass = w.readPassword()
+
+				if _, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
+					log.Error("Failed to decrypt key with given passphrase")
+					return
+				}
+			}
+		}
+		// Establish the gas dynamics to be enforced by the signer
+		fmt.Println()
+		fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget)
+		infos.gasTarget = w.readDefaultFloat(infos.gasTarget)
+
+		fmt.Println()
+		fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice)
+		infos.gasPrice = w.readDefaultFloat(infos.gasPrice)
+	}
+	// Try to deploy the full node on the host
+	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos); err != nil {
+		log.Error("Failed to deploy Ethereum node container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	log.Info("Waiting for node to finish booting")
+	time.Sleep(3 * time.Second)
+
+	w.networkStats(false)
+}

cmd/rlpdump/main.go

@@ -32,6 +32,7 @@ import (
 var (
 	hexMode = flag.String("hex", "", "dump given hex data")
 	noASCII = flag.Bool("noascii", false, "don't print ASCII strings readably")
+	single  = flag.Bool("single", false, "print only the first element, discard the rest")
 )
 
 func init() {
@@ -82,6 +83,9 @@ func main() {
 			break
 		}
 		fmt.Println()
+		if *single {
+			break
+		}
 	}
 }
 

cmd/swarm/cleandb.go

@@ -1,24 +1,23 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
+// go-ethereum is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
 import (
-	"log"
-
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -26,14 +25,14 @@ import (
 func cleandb(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 1 {
-		log.Fatal("need path to chunks database as the first and only argument")
+		utils.Fatalf("Need path to chunks database as the first and only argument")
 	}
 
 	chunkDbPath := args[0]
 	hash := storage.MakeHashFunc("SHA3")
 	dbStore, err := storage.NewDbStore(chunkDbPath, hash, 10000000, 0)
 	if err != nil {
-		log.Fatalf("cannot initialise dbstore: %v", err)
+		utils.Fatalf("Cannot initialise dbstore: %v", err)
 	}
 	dbStore.Cleanup()
 }

cmd/swarm/hash.go

@@ -19,9 +19,9 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"os"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -29,12 +29,11 @@ import (
 func hash(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 1 {
-		log.Fatal("Usage: swarm hash <file name>")
+		utils.Fatalf("Usage: swarm hash <file name>")
 	}
 	f, err := os.Open(args[0])
 	if err != nil {
-		fmt.Println("Error opening file " + args[1])
-		os.Exit(1)
+		utils.Fatalf("Error opening file " + args[1])
 	}
 	defer f.Close()
 
@@ -42,7 +41,7 @@ func hash(ctx *cli.Context) {
 	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
 	key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
 	if err != nil {
-		log.Fatalf("%v\n", err)
+		utils.Fatalf("%v\n", err)
 	} else {
 		fmt.Printf("%v\n", key)
 	}

cmd/swarm/list.go

@@ -0,0 +1,61 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	list, err := client.List(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}

cmd/swarm/main.go

@@ -35,24 +35,20 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	"gopkg.in/urfave/cli.v1"
 )
 
-const (
-	clientIdentifier = "swarm"
-	versionString    = "0.2"
-)
+const clientIdentifier = "swarm"
 
 var (
 	gitCommit        string // Git SHA1 commit hash of the release (set via linker flags)
-	app              = utils.NewApp(gitCommit, "Ethereum Swarm")
 	testbetBootNodes = []string{
 		"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
 		"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
@@ -71,6 +67,10 @@ var (
 		Name:  "bzzaccount",
 		Usage: "Swarm account key file",
 	}
+	SwarmListenAddrFlag = cli.StringFlag{
+		Name:  "httpaddr",
+		Usage: "Swarm HTTP API listening interface",
+	}
 	SwarmPortFlag = cli.StringFlag{
 		Name:  "bzzport",
 		Usage: "Swarm local http api port",
@@ -113,19 +113,36 @@ var (
 		Name:  "defaultpath",
 		Usage: "path to file served for empty url path (none)",
 	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "force mime type",
+	}
 	CorsStringFlag = cli.StringFlag{
 		Name:  "corsdomain",
 		Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
 	}
 )
 
-func init() {
-	// Override flag defaults so bzzd can run alongside geth.
-	utils.ListenPortFlag.Value = 30399
-	utils.IPCPathFlag.Value = utils.DirectoryString{Value: "bzzd.ipc"}
-	utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, web3"
+var defaultNodeConfig = node.DefaultConfig
 
-	// Set up the cli app.
+// This init function sets defaults so cmd/swarm can run alongside geth.
+func init() {
+	defaultNodeConfig.Name = clientIdentifier
+	defaultNodeConfig.Version = params.VersionWithCommit(gitCommit)
+	defaultNodeConfig.P2P.ListenAddr = ":30399"
+	defaultNodeConfig.IPCPath = "bzzd.ipc"
+	// Set flag defaults for --help display.
+	utils.ListenPortFlag.Value = 30399
+}
+
+var app = utils.NewApp(gitCommit, "Ethereum Swarm")
+
+// This init function creates the cli.App.
+func init() {
 	app.Action = bzzd
 	app.HideVersion = true // we have a command to print the version
 	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
@@ -146,6 +163,15 @@ The output of this command is supposed to be machine-readable.
 			ArgsUsage: " <file>",
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
 `,
 		},
 		{
@@ -219,14 +245,15 @@ Cleans database of corrupted entries.
 		utils.MaxPeersFlag,
 		utils.NATFlag,
 		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
 		utils.IPCPathFlag,
+		utils.PasswordFileFlag,
 		// bzzd-specific flags
 		CorsStringFlag,
 		EthAPIFlag,
 		SwarmConfigPathFlag,
 		SwarmSwapEnabledFlag,
 		SwarmSyncEnabledFlag,
+		SwarmListenAddrFlag,
 		SwarmPortFlag,
 		SwarmAccountFlag,
 		SwarmNetworkIdFlag,
@@ -236,6 +263,8 @@ Cleans database of corrupted entries.
 		SwarmRecursiveUploadFlag,
 		SwarmWantManifestFlag,
 		SwarmUploadDefaultPath,
+		SwarmUpFromStdinFlag,
+		SwarmUploadMimeType,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Before = func(ctx *cli.Context) error {
@@ -257,7 +286,7 @@ func main() {
 
 func version(ctx *cli.Context) error {
 	fmt.Println(strings.Title(clientIdentifier))
-	fmt.Println("Version:", versionString)
+	fmt.Println("Version:", params.Version)
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
@@ -270,17 +299,25 @@ func version(ctx *cli.Context) error {
 }
 
 func bzzd(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	cfg := defaultNodeConfig
+	utils.SetNodeConfig(ctx, &cfg)
+	stack, err := node.New(&cfg)
+	if err != nil {
+		utils.Fatalf("can't create node: %v", err)
+	}
+
 	registerBzzService(ctx, stack)
 	utils.StartNode(stack)
+
 	go func() {
 		sigc := make(chan os.Signal, 1)
 		signal.Notify(sigc, syscall.SIGTERM)
 		defer signal.Stop(sigc)
 		<-sigc
-		glog.V(logger.Info).Infoln("Got sigterm, shutting down...")
+		log.Info("Got sigterm, shutting swarm down...")
 		stack.Stop()
 	}()
+
 	networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
 	// Add bootnodes as initial peers.
 	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
@@ -297,7 +334,6 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
-
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -314,6 +350,9 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if len(bzzport) > 0 {
 		bzzconfig.Port = bzzport
 	}
+	if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" {
+		bzzconfig.ListenAddr = bzzaddr
+	}
 	swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
 	syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
 
@@ -327,6 +366,8 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 			if err != nil {
 				utils.Fatalf("Can't connect: %v", err)
 			}
+		} else {
+			swapEnabled = false
 		}
 		return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled, cors)
 	}
@@ -343,17 +384,17 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	}
 	// Try to load the arg as a hex key file.
 	if key, err := crypto.LoadECDSA(keyid); err == nil {
-		glog.V(logger.Info).Infof("swarm account key loaded: %#x", crypto.PubkeyToAddress(key.PublicKey))
+		log.Info("Swarm account key loaded", "address", crypto.PubkeyToAddress(key.PublicKey))
 		return key
 	}
 	// Otherwise try getting it from the keystore.
 	am := stack.AccountManager()
 	ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	return decryptStoreAccount(ks, keyid)
+	return decryptStoreAccount(ks, keyid, utils.MakePasswordList(ctx))
 }
 
-func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKey {
+func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
 	var a accounts.Account
 	var err error
 	if common.IsHexAddress(account) {
@@ -374,9 +415,9 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	if err != nil {
 		utils.Fatalf("Can't load swarm account key: %v", err)
 	}
-	for i := 1; i <= 3; i++ {
-		passphrase := promptPassphrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i))
-		key, err := keystore.DecryptKey(keyjson, passphrase)
+	for i := 0; i < 3; i++ {
+		password := getPassPhrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i+1), i, passwords)
+		key, err := keystore.DecryptKey(keyjson, password)
 		if err == nil {
 			return key.PrivateKey
 		}
@@ -385,7 +426,18 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	return nil
 }
 
-func promptPassphrase(prompt string) string {
+// getPassPhrase retrieves the password associated with bzz account, either by fetching
+// from a list of pre-loaded passwords, or by requesting it interactively from user.
+func getPassPhrase(prompt string, i int, passwords []string) string {
+	// non-interactive
+	if len(passwords) > 0 {
+		if i < len(passwords) {
+			return passwords[i]
+		}
+		return passwords[len(passwords)-1]
+	}
+
+	// fallback to interactive mode
 	if prompt != "" {
 		fmt.Println(prompt)
 	}
@@ -400,7 +452,7 @@ func injectBootnodes(srv *p2p.Server, nodes []string) {
 	for _, url := range nodes {
 		n, err := discover.ParseNode(url)
 		if err != nil {
-			glog.Errorf("invalid bootnode %q", err)
+			log.Error("Invalid swarm bootnode", "err", err)
 			continue
 		}
 		srv.AddPeer(n)

cmd/swarm/manifest.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -18,40 +18,41 @@
 package main
 
 import (
-	"gopkg.in/urfave/cli.v1"
-	"log"
+	"encoding/json"
+	"fmt"
 	"mime"
 	"path/filepath"
 	"strings"
-	"fmt"
-	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
 )
 
 func add(ctx *cli.Context) {
-
 	args := ctx.Args()
 	if len(args) < 3 {
-		log.Fatal("need atleast three arguments <MHASH> <path> <HASH> [<content-type>]")
+		utils.Fatalf("Need atleast three arguments <MHASH> <path> <HASH> [<content-type>]")
 	}
 
 	var (
-		mhash  = args[0]
-		path   = args[1]
-		hash   = args[2]
+		mhash = args[0]
+		path  = args[1]
+		hash  = args[2]
 
-		ctype  string
+		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot  manifest
+		mroot        api.Manifest
 	)
 
-
 	if len(args) > 3 {
 		ctype = args[3]
 	} else {
 		ctype = mime.TypeByExtension(filepath.Ext(path))
 	}
 
-	newManifest := addEntryToManifest (ctx, mhash, path, hash, ctype)
+	newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype)
 	fmt.Println(newManifest)
 
 	if !wantManifest {
@@ -66,17 +67,17 @@ func update(ctx *cli.Context) {
 
 	args := ctx.Args()
 	if len(args) < 3 {
-		log.Fatal("need atleast three arguments <MHASH> <path> <HASH>")
+		utils.Fatalf("Need atleast three arguments <MHASH> <path> <HASH>")
 	}
 
 	var (
-		mhash  = args[0]
-		path   = args[1]
-		hash   = args[2]
+		mhash = args[0]
+		path  = args[1]
+		hash  = args[2]
 
-		ctype  string
+		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot  manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -84,7 +85,7 @@ func update(ctx *cli.Context) {
 		ctype = mime.TypeByExtension(filepath.Ext(path))
 	}
 
-	newManifest := updateEntryInManifest (ctx, mhash, path, hash, ctype)
+	newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype)
 	fmt.Println(newManifest)
 
 	if !wantManifest {
@@ -98,18 +99,18 @@ func update(ctx *cli.Context) {
 func remove(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 2 {
-		log.Fatal("need atleast two arguments <MHASH> <path>")
+		utils.Fatalf("Need atleast two arguments <MHASH> <path>")
 	}
 
 	var (
-		mhash  = args[0]
-		path   = args[1]
+		mhash = args[0]
+		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot  manifest
+		mroot        api.Manifest
 	)
 
-	newManifest := removeEntryFromManifest (ctx, mhash, path)
+	newManifest := removeEntryFromManifest(ctx, mhash, path)
 	fmt.Println(newManifest)
 
 	if !wantManifest {
@@ -120,35 +121,30 @@ func remove(ctx *cli.Context) {
 	}
 }
 
-func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string)  string {
+func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = &client{api: bzzapi}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType:  "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
-		log.Fatalln("manifest download failed:", err)
+		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.downloadManifest(hash)
+	_, err = client.DownloadManifest(hash)
 	if err != nil {
-		log.Fatalln("hash to add is not present:", err)
+		utils.Fatalf("Hash to add is not present: %v", err)
 	}
 
-
 	// See if we path is in this Manifest or do we have to dig deeper
 	for _, entry := range mroot.Entries {
 		if path == entry.Path {
-			log.Fatal(path, "Already present, not adding anything")
-		}else {
+			utils.Fatalf("Path %s already present, not adding anything", path)
+		} else {
 			if entry.ContentType == "application/bzz-manifest+json" {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
@@ -161,10 +157,10 @@ func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string)  st
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest add the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := addEntryToManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype)
+		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -174,55 +170,43 @@ func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string)  st
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := manifestEntry{
-			Path:        path,
+		newEntry := api.ManifestEntry{
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
-		log.Fatalln("manifest upload failed:", err)
+		utils.Fatalf("Manifest upload failed: %v", err)
 	}
 	return newManifestHash
 
-
-
 }
 
-func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string) string {
+func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = &client{api: bzzapi}
-		newEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType:  "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType:  "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		newEntry         = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
-		log.Fatalln("manifest download failed:", err)
+		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" with which to update is valid and present in swarm
 
-
 	// See if we path is in this Manifest or do we have to dig deeper
 	for _, entry := range mroot.Entries {
 		if path == entry.Path {
 			newEntry = entry
-		}else {
+		} else {
 			if entry.ContentType == "application/bzz-manifest+json" {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
@@ -233,16 +217,16 @@ func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string)
 	}
 
 	if longestPathEntry.Path == "" && newEntry.Path == "" {
-		log.Fatal(path, " Path not present in the Manifest, not setting anything")
+		utils.Fatalf("Path %s not present in the Manifest, not setting anything", path)
 	}
 
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest add the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := updateEntryInManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype)
+		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -255,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string)
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := manifestEntry{
-					Path:        entry.Path,
+				myEntry := api.ManifestEntry{
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -271,43 +255,32 @@ func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string)
 		mroot = newMRoot
 	}
 
-
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
-		log.Fatalln("manifest upload failed:", err)
+		utils.Fatalf("Manifest upload failed: %v", err)
 	}
 	return newManifestHash
 }
 
-func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string {
+func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = &client{api: bzzapi}
-		entryToRemove = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType:  "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType:  "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		entryToRemove    = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
-		log.Fatalln("manifest download failed:", err)
+		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
-
-
 	// See if we path is in this Manifest or do we have to dig deeper
 	for _, entry := range mroot.Entries {
 		if path == entry.Path {
 			entryToRemove = entry
-		}else {
+		} else {
 			if entry.ContentType == "application/bzz-manifest+json" {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
@@ -318,16 +291,16 @@ func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string {
 	}
 
 	if longestPathEntry.Path == "" && entryToRemove.Path == "" {
-		log.Fatal(path, "Path not present in the Manifest, not removing anything")
+		utils.Fatalf("Path %s not present in the Manifest, not removing anything", path)
 	}
 
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest remove the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := removeEntryFromManifest (ctx, longestPathEntry.Hash, newPath)
+		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -339,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -348,13 +321,9 @@ func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string {
 		mroot = newMRoot
 	}
 
-
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
-		log.Fatalln("manifest upload failed:", err)
+		utils.Fatalf("Manifest upload failed: %v", err)
 	}
 	return newManifestHash
-
-
 }
-

cmd/swarm/upload.go

@@ -18,12 +18,9 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"mime"
 	"net/http"
 	"os"
@@ -32,57 +29,84 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
 func upload(ctx *cli.Context) {
+
 	args := ctx.Args()
 	var (
 		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		recursive    = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
+		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
+
 	if len(args) != 1 {
-		log.Fatal("need filename as the first and only argument")
+		if fromStdin {
+			tmp, err := ioutil.TempFile("", "swarm-stdin")
+			if err != nil {
+				utils.Fatalf("error create tempfile: %s", err)
+			}
+			defer os.Remove(tmp.Name())
+			n, err := io.Copy(tmp, os.Stdin)
+			if err != nil {
+				utils.Fatalf("error copying stdin to tempfile: %s", err)
+			} else if n == 0 {
+				utils.Fatalf("error reading from stdin: zero length")
+			}
+			file = tmp.Name()
+		} else {
+			utils.Fatalf("Need filename as the first and only argument")
+		}
+	} else {
+		file = expandPath(args[0])
 	}
 
-	var (
-		file   = args[0]
-		client = &client{api: bzzapi}
-	)
-	fi, err := os.Stat(expandPath(file))
-	if err != nil {
-		log.Fatal(err)
-	}
-	if fi.IsDir() {
-		if !recursive {
-			log.Fatal("argument is a directory and recursive upload is disabled")
-		}
-		if !wantManifest {
-			log.Fatal("manifest is required for directory uploads")
-		}
-		mhash, err := client.uploadDirectory(file, defaultPath)
-		if err != nil {
-			log.Fatal(err)
-		}
-		fmt.Println(mhash)
-		return
-	}
-	entry, err := client.uploadFile(file, fi)
-	if err != nil {
-		log.Fatalln("upload failed:", err)
-	}
-	mroot := manifest{[]manifestEntry{entry}}
 	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
 		return
 	}
-	hash, err := client.uploadManifest(mroot)
+
+	stat, err := os.Stat(file)
 	if err != nil {
-		log.Fatalln("manifest upload failed:", err)
+		utils.Fatalf("Error opening file: %s", err)
+	}
+	var hash string
+	if stat.IsDir() {
+		if !recursive {
+			utils.Fatalf("Argument is a directory and recursive upload is disabled")
+		}
+		hash, err = client.UploadDirectory(file, defaultPath, "")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
+		}
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
+	}
+	if err != nil {
+		utils.Fatalf("Upload failed: %s", err)
 	}
 	fmt.Println(hash)
 }
@@ -111,147 +135,18 @@ func homeDir() string {
 	return ""
 }
 
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
-	api string
+func detectMimeType(file string) string {
+	if ext := filepath.Ext(file); ext != "" {
+		return mime.TypeByExtension(ext)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	buf := make([]byte, 512)
+	if n, _ := f.Read(buf); n > 0 {
+		return http.DetectContentType(buf)
+	}
+	return ""
 }
-
-// manifest is the JSON representation of a swarm manifest.
-type manifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
-	Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
-	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
-	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Printf("uploading file %s (%d bytes)", file, fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("uploading manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Printf("uploading file %s (%d bytes) and adding path %v", fpath, fi.Size(), path)
-	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
-	mroot := manifest{}
-	req, err := http.NewRequest("GET", c.api + "/bzzr:/" + mhash, nil)
-	if err != nil {
-		return mroot, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return mroot, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-
-	err = json.Unmarshal(content, &mroot)
-	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
-	}
-	return mroot, err
-}

cmd/utils/cmd.go

@@ -23,16 +23,13 @@ import (
 	"io"
 	"os"
 	"os/signal"
-	"regexp"
 	"runtime"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -41,15 +38,6 @@ const (
 	importBatchSize = 2500
 )
 
-func openLogFile(Datadir string, filename string) *os.File {
-	path := common.AbsolutePath(Datadir, filename)
-	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-	if err != nil {
-		panic(fmt.Sprintf("error opening log file '%s': %v", filename, err))
-	}
-	return file
-}
-
 // Fatalf formats a message to standard error and exits the program.
 // The message is also printed to standard output if standard error
 // is redirected to a different file.
@@ -79,12 +67,12 @@ func StartNode(stack *node.Node) {
 		signal.Notify(sigc, os.Interrupt)
 		defer signal.Stop(sigc)
 		<-sigc
-		glog.V(logger.Info).Infoln("Got interrupt, shutting down...")
+		log.Info("Got interrupt, shutting down...")
 		go stack.Stop()
 		for i := 10; i > 0; i-- {
 			<-sigc
 			if i > 1 {
-				glog.V(logger.Info).Infof("Already shutting down, interrupt %d more times for panic.", i-1)
+				log.Warn("Already shutting down, interrupt more to panic.", "times", i-1)
 			}
 		}
 		debug.Exit() // ensure trace and CPU profile data is flushed.
@@ -92,19 +80,6 @@ func StartNode(stack *node.Node) {
 	}()
 }
 
-func FormatTransactionData(data string) []byte {
-	d := common.StringToByteFunc(data, func(s string) (ret []byte) {
-		slice := regexp.MustCompile(`\n|\s`).Split(s, 1000000000)
-		for _, dataItem := range slice {
-			d := common.FormatData(dataItem)
-			ret = append(ret, d...)
-		}
-		return
-	})
-
-	return d
-}
-
 func ImportChain(chain *core.BlockChain, fn string) error {
 	// Watch for Ctrl-C while the import is running.
 	// If a signal is received, the import will stop at the next batch.
@@ -115,7 +90,7 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 	defer close(interrupt)
 	go func() {
 		if _, ok := <-interrupt; ok {
-			glog.Info("caught interrupt during import, will stop at next batch")
+			log.Info("Interrupted during import, stopping at next batch")
 		}
 		close(stop)
 	}()
@@ -128,7 +103,7 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 		}
 	}
 
-	glog.Infoln("Importing blockchain ", fn)
+	log.Info("Importing blockchain", "file", fn)
 	fh, err := os.Open(fn)
 	if err != nil {
 		return err
@@ -176,8 +151,7 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 			return fmt.Errorf("interrupted")
 		}
 		if hasAllBlocks(chain, blocks[:i]) {
-			glog.Infof("skipping batch %d, all blocks present [%x / %x]",
-				batch, blocks[0].Hash().Bytes()[:4], blocks[i-1].Hash().Bytes()[:4])
+			log.Info("Skipping batch as all blocks present", "batch", batch, "first", blocks[0].Hash(), "last", blocks[i-1].Hash())
 			continue
 		}
 
@@ -198,7 +172,7 @@ func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool {
 }
 
 func ExportChain(blockchain *core.BlockChain, fn string) error {
-	glog.Infoln("Exporting blockchain to ", fn)
+	log.Info("Exporting blockchain", "file", fn)
 	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
 	if err != nil {
 		return err
@@ -214,13 +188,13 @@ func ExportChain(blockchain *core.BlockChain, fn string) error {
 	if err := blockchain.Export(writer); err != nil {
 		return err
 	}
-	glog.Infoln("Exported blockchain to ", fn)
+	log.Info("Exported blockchain", "file", fn)
 
 	return nil
 }
 
 func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, last uint64) error {
-	glog.Infoln("Exporting blockchain to ", fn)
+	log.Info("Exporting blockchain", "file", fn)
 	// TODO verify mode perms
 	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
 	if err != nil {
@@ -237,6 +211,6 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
 	if err := blockchain.ExportN(writer, first, last); err != nil {
 		return err
 	}
-	glog.Infoln("Exported blockchain to ", fn)
+	log.Info("Exported blockchain to", "file", fn)
 	return nil
 }

cmd/utils/customflags.go

@@ -17,12 +17,18 @@
 package utils
 
 import (
+	"encoding"
+	"errors"
 	"flag"
 	"fmt"
+	"math/big"
 	"os"
 	"os/user"
 	"path"
 	"strings"
+
+	"github.com/ethereum/go-ethereum/common/math"
+	"gopkg.in/urfave/cli.v1"
 )
 
 // Custom type which is registered in the flags library which cli uses for
@@ -44,10 +50,9 @@ func (self *DirectoryString) Set(value string) error {
 // Custom cli.Flag type which expand the received string to an absolute path.
 // e.g. ~/.ethereum -> /home/username/.ethereum
 type DirectoryFlag struct {
-	Name   string
-	Value  DirectoryString
-	Usage  string
-	EnvVar string
+	Name  string
+	Value DirectoryString
+	Usage string
 }
 
 func (self DirectoryFlag) String() string {
@@ -55,7 +60,7 @@ func (self DirectoryFlag) String() string {
 	if len(self.Value.Value) > 0 {
 		fmtString = "%s \"%v\"\t%v"
 	}
-	return withEnvHint(self.EnvVar, fmt.Sprintf(fmtString, prefixedNames(self.Name), self.Value.Value, self.Usage))
+	return fmt.Sprintf(fmtString, prefixedNames(self.Name), self.Value.Value, self.Usage)
 }
 
 func eachName(longName string, fn func(string)) {
@@ -69,21 +74,117 @@ func eachName(longName string, fn func(string)) {
 // called by cli library, grabs variable from environment (if in env)
 // and adds variable to flag set for parsing.
 func (self DirectoryFlag) Apply(set *flag.FlagSet) {
-	if self.EnvVar != "" {
-		for _, envVar := range strings.Split(self.EnvVar, ",") {
-			envVar = strings.TrimSpace(envVar)
-			if envVal := os.Getenv(envVar); envVal != "" {
-				self.Value.Value = envVal
-				break
-			}
-		}
-	}
-
 	eachName(self.Name, func(name string) {
 		set.Var(&self.Value, self.Name, self.Usage)
 	})
 }
 
+type TextMarshaler interface {
+	encoding.TextMarshaler
+	encoding.TextUnmarshaler
+}
+
+// textMarshalerVal turns a TextMarshaler into a flag.Value
+type textMarshalerVal struct {
+	v TextMarshaler
+}
+
+func (v textMarshalerVal) String() string {
+	if v.v == nil {
+		return ""
+	}
+	text, _ := v.v.MarshalText()
+	return string(text)
+}
+
+func (v textMarshalerVal) Set(s string) error {
+	return v.v.UnmarshalText([]byte(s))
+}
+
+// TextMarshalerFlag wraps a TextMarshaler value.
+type TextMarshalerFlag struct {
+	Name  string
+	Value TextMarshaler
+	Usage string
+}
+
+func (f TextMarshalerFlag) GetName() string {
+	return f.Name
+}
+
+func (f TextMarshalerFlag) String() string {
+	return fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+}
+
+func (f TextMarshalerFlag) Apply(set *flag.FlagSet) {
+	eachName(f.Name, func(name string) {
+		set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage)
+	})
+}
+
+// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set.
+func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler {
+	val := ctx.GlobalGeneric(name)
+	if val == nil {
+		return nil
+	}
+	return val.(textMarshalerVal).v
+}
+
+// BigFlag is a command line flag that accepts 256 bit big integers in decimal or
+// hexadecimal syntax.
+type BigFlag struct {
+	Name  string
+	Value *big.Int
+	Usage string
+}
+
+// bigValue turns *big.Int into a flag.Value
+type bigValue big.Int
+
+func (b *bigValue) String() string {
+	if b == nil {
+		return ""
+	}
+	return (*big.Int)(b).String()
+}
+
+func (b *bigValue) Set(s string) error {
+	int, ok := math.ParseBig256(s)
+	if !ok {
+		return errors.New("invalid integer syntax")
+	}
+	*b = (bigValue)(*int)
+	return nil
+}
+
+func (f BigFlag) GetName() string {
+	return f.Name
+}
+
+func (f BigFlag) String() string {
+	fmtString := "%s %v\t%v"
+	if f.Value != nil {
+		fmtString = "%s \"%v\"\t%v"
+	}
+	return fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage)
+}
+
+func (f BigFlag) Apply(set *flag.FlagSet) {
+	eachName(f.Name, func(name string) {
+		set.Var((*bigValue)(f.Value), f.Name, f.Usage)
+	})
+}
+
+// GlobalBig returns the value of a BigFlag from the global flag set.
+func GlobalBig(ctx *cli.Context, name string) *big.Int {
+	val := ctx.GlobalGeneric(name)
+	if val == nil {
+		return nil
+	}
+	return (*big.Int)(val.(*bigValue))
+}
+
 func prefixFor(name string) (prefix string) {
 	if len(name) == 1 {
 		prefix = "-"
@@ -106,14 +207,6 @@ func prefixedNames(fullName string) (prefixed string) {
 	return
 }
 
-func withEnvHint(envVar, str string) string {
-	envText := ""
-	if envVar != "" {
-		envText = fmt.Sprintf(" [$%s]", strings.Join(strings.Split(envVar, ","), ", $"))
-	}
-	return str + envText
-}
-
 func (self DirectoryFlag) GetName() string {
 	return self.Name
 }

cmd/wnode/main.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -22,14 +22,14 @@ package main
 import (
 	"bufio"
 	"crypto/ecdsa"
-	"crypto/sha1"
-	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/binary"
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -38,8 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/nat"
@@ -67,33 +66,38 @@ var (
 	asymKey    *ecdsa.PrivateKey
 	nodeid     *ecdsa.PrivateKey
 	topic      whisper.TopicType
-	filterID   uint32
+	asymKeyID  string
+	filterID   string
+	symPass    string
 	msPassword string
 )
 
 // cmd arguments
 var (
-	echoMode       = flag.Bool("e", false, "echo mode: prints some arguments for diagnostics")
-	bootstrapMode  = flag.Bool("b", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
-	forwarderMode  = flag.Bool("f", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
-	mailServerMode = flag.Bool("s", false, "mail server mode: delivers expired messages on demand")
-	requestMail    = flag.Bool("r", false, "request expired messages from the bootstrap server")
-	asymmetricMode = flag.Bool("a", false, "use asymmetric encryption")
-	testMode       = flag.Bool("t", false, "use of predefined parameters for diagnostics")
-	generateKey    = flag.Bool("k", false, "generate and show the private key")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
+	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
+	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
+	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
+	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
+	argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level")
 	argTTL       = flag.Uint("ttl", 30, "time-to-live for messages in seconds")
 	argWorkTime  = flag.Uint("work", 5, "work time in seconds")
-	argPoW       = flag.Float64("pow", whisper.MinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
-	argServerPoW = flag.Float64("mspow", whisper.MinimumPoW, "PoW requirement for Mail Server request")
+	argMaxSize   = flag.Int("maxsize", whisper.DefaultMaxMessageLength, "max size of message")
+	argPoW       = flag.Float64("pow", whisper.DefaultMinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argServerPoW = flag.Float64("mspow", whisper.DefaultMinimumPoW, "PoW requirement for Mail Server request")
 
-	argIP     = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:30303)")
-	argSalt   = flag.String("salt", "", "salt (for topic and key derivation)")
-	argPub    = flag.String("pub", "", "public key for asymmetric encryption")
-	argDBPath = flag.String("dbpath", "", "path to the server's DB directory")
-	argIDFile = flag.String("idfile", "", "file name with node id (private key)")
-	argEnode  = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
-	argTopic  = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+	argIP      = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:30303)")
+	argPub     = flag.String("pub", "", "public key for asymmetric encryption")
+	argDBPath  = flag.String("dbpath", "", "path to the server's DB directory")
+	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
+	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
+	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
 )
 
 func main() {
@@ -135,6 +139,14 @@ func processArgs() {
 		}
 	}
 
+	if len(*argSaveDir) > 0 {
+		if _, err := os.Stat(*argSaveDir); os.IsNotExist(err) {
+			utils.Fatalf("Download directory '%s' does not exist", *argSaveDir)
+		}
+	} else if *fileExMode {
+		utils.Fatalf("Parameter 'savedir' is mandatory for file exchange mode")
+	}
+
 	if *echoMode {
 		echo()
 	}
@@ -146,7 +158,6 @@ func echo() {
 	fmt.Printf("pow = %f \n", *argPoW)
 	fmt.Printf("mspow = %f \n", *argServerPoW)
 	fmt.Printf("ip = %s \n", *argIP)
-	fmt.Printf("salt = %s \n", *argSalt)
 	fmt.Printf("pub = %s \n", common.ToHex(crypto.FromECDSAPub(pub)))
 	fmt.Printf("idfile = %s \n", *argIDFile)
 	fmt.Printf("dbpath = %s \n", *argDBPath)
@@ -154,8 +165,7 @@ func echo() {
 }
 
 func initialize() {
-	glog.SetV(logger.Warn)
-	glog.SetToStderr(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*argVerbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
 
 	done = make(chan struct{})
 	var peers []*discover.Node
@@ -172,11 +182,8 @@ func initialize() {
 	}
 
 	if *testMode {
-		password := []byte("test password for symmetric encryption")
-		salt := []byte("test salt for symmetric encryption")
-		symKey = pbkdf2.Key(password, salt, 64, 32, sha256.New)
-		topic = whisper.TopicType{0xFF, 0xFF, 0xFF, 0xFF}
-		msPassword = "mail server test password"
+		symPass = "wwww" // ascii code: 0x77777777
+		msPassword = "wwww"
 	}
 
 	if *bootstrapMode {
@@ -198,15 +205,47 @@ func initialize() {
 				utils.Fatalf("Failed to read Mail Server password: %s", err)
 			}
 		}
-		shh = whisper.NewWhisper(&mailServer)
+		shh = whisper.New()
+		shh.RegisterServer(&mailServer)
 		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
 	} else {
-		shh = whisper.NewWhisper(nil)
+		shh = whisper.New()
+	}
+
+	if *argPoW != whisper.DefaultMinimumPoW {
+		err := shh.SetMinimumPoW(*argPoW)
+		if err != nil {
+			utils.Fatalf("Failed to set PoW: %s", err)
+		}
+	}
+
+	if *argMaxSize != whisper.DefaultMaxMessageLength {
+		err := shh.SetMaxMessageLength(*argMaxSize)
+		if err != nil {
+			utils.Fatalf("Failed to set max message size: %s", err)
+		}
+	}
+
+	asymKeyID, err = shh.NewKeyPair()
+	if err != nil {
+		utils.Fatalf("Failed to generate a new key pair: %s", err)
+	}
+
+	asymKey, err = shh.GetPrivateKey(asymKeyID)
+	if err != nil {
+		utils.Fatalf("Failed to retrieve a new key pair: %s", err)
 	}
 
-	asymKey = shh.NewIdentity()
 	if nodeid == nil {
-		nodeid = shh.NewIdentity()
+		tmpID, err := shh.NewKeyPair()
+		if err != nil {
+			utils.Fatalf("Failed to generate a new key pair: %s", err)
+		}
+
+		nodeid, err = shh.GetPrivateKey(tmpID)
+		if err != nil {
+			utils.Fatalf("Failed to retrieve a new key pair: %s", err)
+		}
 	}
 
 	maxPeers := 80
@@ -218,7 +257,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("whisper-go", "5.0"),
+			Name:           common.MakeName("wnode", "5.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -268,7 +307,11 @@ func configureNode() {
 	if *asymmetricMode {
 		if len(*argPub) == 0 {
 			s := scanLine("Please enter the peer's public key: ")
-			pub = crypto.ToECDSAPub(common.FromHex(s))
+			b := common.FromHex(s)
+			if b == nil {
+				utils.Fatalf("Error: can not convert hexadecimal string")
+			}
+			pub = crypto.ToECDSAPub(b)
 			if !isKeyValid(pub) {
 				utils.Fatalf("Error: invalid public key")
 			}
@@ -285,21 +328,27 @@ func configureNode() {
 		}
 	}
 
-	if !*asymmetricMode && !*forwarderMode && !*testMode {
-		pass, err := console.Stdin.PromptPassword("Please enter the password: ")
+	if !*asymmetricMode && !*forwarderMode {
+		if len(symPass) == 0 {
+			symPass, err = console.Stdin.PromptPassword("Please enter the password for symmetric encryption: ")
+			if err != nil {
+				utils.Fatalf("Failed to read passphrase: %v", err)
+			}
+		}
+
+		symKeyID, err := shh.AddSymKeyFromPassword(symPass)
 		if err != nil {
-			utils.Fatalf("Failed to read passphrase: %v", err)
+			utils.Fatalf("Failed to create symmetric key: %s", err)
 		}
-
-		if len(*argSalt) == 0 {
-			argSalt = scanLineA("Please enter the salt: ")
+		symKey, err = shh.GetSymKey(symKeyID)
+		if err != nil {
+			utils.Fatalf("Failed to save symmetric key: %s", err)
 		}
-
-		symKey = pbkdf2.Key([]byte(pass), []byte(*argSalt), 65356, 32, sha256.New)
-
 		if len(*argTopic) == 0 {
-			generateTopic([]byte(pass), []byte(*argSalt))
+			generateTopic([]byte(symPass))
 		}
+
+		fmt.Printf("Filter is configured for the topic: %x \n", topic)
 	}
 
 	if *mailServerMode {
@@ -309,24 +358,21 @@ func configureNode() {
 	}
 
 	filter := whisper.Filter{
-		KeySym:    symKey,
-		KeyAsym:   asymKey,
-		Topics:    []whisper.TopicType{topic},
-		AcceptP2P: p2pAccept,
+		KeySym:   symKey,
+		KeyAsym:  asymKey,
+		Topics:   [][]byte{topic[:]},
+		AllowP2P: p2pAccept,
+	}
+	filterID, err = shh.Subscribe(&filter)
+	if err != nil {
+		utils.Fatalf("Failed to install filter: %s", err)
 	}
-	filterID = shh.Watch(&filter)
-	fmt.Printf("Filter is configured for the topic: %x \n", topic)
 }
 
-func generateTopic(password, salt []byte) {
-	const rounds = 4000
-	const size = 128
-	x1 := pbkdf2.Key(password, salt, rounds, size, sha512.New)
-	x2 := pbkdf2.Key(password, salt, rounds, size, sha1.New)
-	x3 := pbkdf2.Key(x1, x2, rounds, size, sha256.New)
-
-	for i := 0; i < size; i++ {
-		topic[i%whisper.TopicLength] ^= x3[i]
+func generateTopic(password []byte) {
+	x := pbkdf2.Key(password, password, 4096, 128, sha512.New)
+	for i := 0; i < len(x); i++ {
+		topic[i%whisper.TopicLength] ^= x[i]
 	}
 }
 
@@ -360,6 +406,8 @@ func run() {
 
 	if *requestMail {
 		requestExpiredMessagesLoop()
+	} else if *fileExMode {
+		sendFilesLoop()
 	} else {
 		sendLoop()
 	}
@@ -378,9 +426,34 @@ func sendLoop() {
 		if *asymmetricMode {
 			// print your own message for convenience,
 			// because in asymmetric mode it is impossible to decrypt it
-			hour, min, sec := time.Now().Clock()
+			timestamp := time.Now().Unix()
 			from := crypto.PubkeyToAddress(asymKey.PublicKey)
-			fmt.Printf("\n%02d:%02d:%02d <%x>: %s\n", hour, min, sec, from, s)
+			fmt.Printf("\n%d <%x>: %s\n", timestamp, from, s)
+		}
+	}
+}
+
+func sendFilesLoop() {
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			close(done)
+			break
+		}
+		b, err := ioutil.ReadFile(s)
+		if err != nil {
+			fmt.Printf(">>> Error: %s \n", err)
+			continue
+		} else {
+			h := sendMsg(b)
+			if (h == common.Hash{}) {
+				fmt.Printf(">>> Error: message was not sent \n")
+			} else {
+				timestamp := time.Now().Unix()
+				from := crypto.PubkeyToAddress(asymKey.PublicKey)
+				fmt.Printf("\n%d <%x>: sent message with hash %x\n", timestamp, from, h)
+			}
 		}
 	}
 }
@@ -411,7 +484,7 @@ func scanUint(prompt string) uint32 {
 	return uint32(i)
 }
 
-func sendMsg(payload []byte) {
+func sendMsg(payload []byte) common.Hash {
 	params := whisper.MessageParams{
 		Src:      asymKey,
 		Dst:      pub,
@@ -423,17 +496,23 @@ func sendMsg(payload []byte) {
 		WorkTime: uint32(*argWorkTime),
 	}
 
-	msg := whisper.NewSentMessage(&params)
+	msg, err := whisper.NewSentMessage(&params)
+	if err != nil {
+		utils.Fatalf("failed to create new message: %s", err)
+	}
 	envelope, err := msg.Wrap(&params)
 	if err != nil {
 		fmt.Printf("failed to seal message: %v \n", err)
-		return
+		return common.Hash{}
 	}
 
 	err = shh.Send(envelope)
 	if err != nil {
 		fmt.Printf("failed to send message: %v \n", err)
+		return common.Hash{}
 	}
+
+	return envelope.Hash()
 }
 
 func messageLoop() {
@@ -449,7 +528,11 @@ func messageLoop() {
 		case <-ticker.C:
 			messages := f.Retrieve()
 			for _, msg := range messages {
-				printMessageInfo(msg)
+				if *fileExMode || len(msg.Payload) > 2048 {
+					writeMessageToFile(*argSaveDir, msg)
+				} else {
+					printMessageInfo(msg)
+				}
 			}
 		case <-done:
 			return
@@ -473,19 +556,47 @@ func printMessageInfo(msg *whisper.ReceivedMessage) {
 	}
 }
 
+func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
+	timestamp := fmt.Sprintf("%d", msg.Sent)
+	name := fmt.Sprintf("%x", msg.EnvelopeHash)
+
+	var address common.Address
+	if msg.Src != nil {
+		address = crypto.PubkeyToAddress(*msg.Src)
+	}
+
+	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+		// message from myself: don't save, only report
+		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
+	} else if len(dir) > 0 {
+		fullpath := filepath.Join(dir, name)
+		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		if err != nil {
+			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
+		} else {
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+		}
+	} else {
+		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+	}
+}
+
 func requestExpiredMessagesLoop() {
 	var key, peerID []byte
 	var timeLow, timeUpp uint32
 	var t string
 	var xt, empty whisper.TopicType
 
-	err := shh.AddSymKey(mailserver.MailServerKeyName, []byte(msPassword))
+	keyID, err := shh.AddSymKeyFromPassword(msPassword)
 	if err != nil {
 		utils.Fatalf("Failed to create symmetric key for mail request: %s", err)
 	}
-	key = shh.GetSymKey(mailserver.MailServerKeyName)
+	key, err = shh.GetSymKey(keyID)
+	if err != nil {
+		utils.Fatalf("Failed to save symmetric key for mail request: %s", err)
+	}
 	peerID = extractIdFromEnode(*argEnode)
-	shh.MarkPeerTrusted(peerID)
+	shh.AllowP2PMessagesFromPeer(peerID)
 
 	for {
 		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
@@ -517,7 +628,10 @@ func requestExpiredMessagesLoop() {
 		params.Src = nodeid
 		params.WorkTime = 5
 
-		msg := whisper.NewSentMessage(&params)
+		msg, err := whisper.NewSentMessage(&params)
+		if err != nil {
+			utils.Fatalf("failed to create new message: %s", err)
+		}
 		env, err := msg.Wrap(&params)
 		if err != nil {
 			utils.Fatalf("Wrap failed: %s", err)
@@ -536,7 +650,6 @@ func extractIdFromEnode(s string) []byte {
 	n, err := discover.ParseNode(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)
-		return nil
 	}
 	return n.ID[:]
 }

common/.gitignore

@@ -1,12 +0,0 @@
-# See http://help.github.com/ignore-files/ for more about ignoring files.
-#
-# If you find yourself ignoring temporary files generated by your text editor
-# or operating system, you probably want to add a global ignore instead:
-#   git config --global core.excludesfile ~/.gitignore_global
-
-/tmp
-*/**/*un~
-*un~
-.DS_Store
-*/**/.DS_Store
-

common/.travis.yml

@@ -1,3 +0,0 @@
-language: go
-go:
-  - 1.2

common/README.md

@@ -1,140 +0,0 @@
-# common
-
-[![Build
-Status](https://travis-ci.org/ethereum/go-ethereum.png?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
-
-The common package contains the ethereum utility library.
-
-# Installation
-
-As a subdirectory the main go-ethereum repository, you get it with
-`go get github.com/ethereum/go-ethereum`.
-
-# Usage
-
-## RLP (Recursive Linear Prefix) Encoding
-
-RLP Encoding is an encoding scheme used by the Ethereum project. It
-encodes any native value or list to a string.
-
-More in depth information about the encoding scheme see the
-[Wiki](http://wiki.ethereum.org/index.php/RLP) article.
-
-```go
-rlp := common.Encode("doge")
-fmt.Printf("%q\n", rlp) // => "\0x83dog"
-
-rlp = common.Encode([]interface{}{"dog", "cat"})
-fmt.Printf("%q\n", rlp) // => "\0xc8\0x83dog\0x83cat"
-decoded := common.Decode(rlp)
-fmt.Println(decoded) // => ["dog" "cat"]
-```
-
-## Patricia Trie
-
-Patricie Tree is a merkle trie used by the Ethereum project.
-
-More in depth information about the (modified) Patricia Trie can be
-found on the [Wiki](http://wiki.ethereum.org/index.php/Patricia_Tree).
-
-The patricia trie uses a db as backend and could be anything as long as
-it satisfies the Database interface found in `common/db.go`.
-
-```go
-db := NewDatabase()
-
-// db, root
-trie := common.NewTrie(db, "")
-
-trie.Put("puppy", "dog")
-trie.Put("horse", "stallion")
-trie.Put("do", "verb")
-trie.Put("doge", "coin")
-
-// Look up the key "do" in the trie
-out := trie.Get("do")
-fmt.Println(out) // => verb
-
-trie.Delete("puppy")
-```
-
-The patricia trie, in combination with RLP, provides a robust,
-cryptographically authenticated data structure that can be used to store
-all (key, value) bindings.
-
-```go
-// ... Create db/trie
-
-// Note that RLP uses interface slices as list
-value := common.Encode([]interface{}{"one", 2, "three", []interface{}{42}})
-// Store the RLP encoded value of the list
-trie.Put("mykey", value)
-```
-
-## Value
-
-Value is a Generic Value which is used in combination with RLP data or
-`([])interface{}` structures. It may serve as a bridge between RLP data
-and actual real values and takes care of all the type checking and
-casting. Unlike Go's `reflect.Value` it does not panic if it's unable to
-cast to the requested value. It simple returns the base value of that
-type (e.g. `Slice()` returns []interface{}, `Uint()` return 0, etc).
-
-### Creating a new Value
-
-`NewEmptyValue()` returns a new \*Value with it's initial value set to a
-`[]interface{}`
-
-`AppendList()` appends a list to the current value.
-
-`Append(v)` appends the value (v) to the current value/list.
-
-```go
-val := common.NewEmptyValue().Append(1).Append("2")
-val.AppendList().Append(3)
-```
-
-### Retrieving values
-
-`Get(i)` returns the `i` item in the list.
-
-`Uint()` returns the value as an unsigned int64.
-
-`Slice()` returns the value as a interface slice.
-
-`Str()` returns the value as a string.
-
-`Bytes()` returns the value as a byte slice.
-
-`Len()` assumes current to be a slice and returns its length.
-
-`Byte()` returns the value as a single byte.
-
-```go
-val := common.NewValue([]interface{}{1,"2",[]interface{}{3}})
-val.Get(0).Uint() // => 1
-val.Get(1).Str()  // => "2"
-s := val.Get(2)   // => Value([]interface{}{3})
-s.Get(0).Uint()   // => 3
-```
-
-## Decoding
-
-Decoding streams of RLP data is simplified
-
-```go
-val := common.NewValueFromBytes(rlpData)
-val.Get(0).Uint()
-```
-
-## Encoding
-
-Encoding from Value to RLP is done with the `Encode` method. The
-underlying value can be anything RLP can encode (int, str, lists, bytes)
-
-```go
-val := common.NewValue([]interface{}{1,"2",[]interface{}{3}})
-rlp := val.Encode()
-// Store the rlp data
-Store(rlp)
-```

common/big.go

@@ -20,137 +20,11 @@ import "math/big"
 
 // Common big integers often used
 var (
-	Big1     = big.NewInt(1)
-	Big2     = big.NewInt(2)
-	Big3     = big.NewInt(3)
-	Big0     = big.NewInt(0)
-	BigTrue  = Big1
-	BigFalse = Big0
-	Big32    = big.NewInt(32)
-	Big36    = big.NewInt(36)
-	Big97    = big.NewInt(97)
-	Big98    = big.NewInt(98)
-	Big256   = big.NewInt(0xff)
-	Big257   = big.NewInt(257)
-	MaxBig   = String2Big("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+	Big1   = big.NewInt(1)
+	Big2   = big.NewInt(2)
+	Big3   = big.NewInt(3)
+	Big0   = big.NewInt(0)
+	Big32  = big.NewInt(32)
+	Big256 = big.NewInt(0xff)
+	Big257 = big.NewInt(257)
 )
-
-// Big pow
-//
-// Returns the power of two big integers
-func BigPow(a, b int) *big.Int {
-	c := new(big.Int)
-	c.Exp(big.NewInt(int64(a)), big.NewInt(int64(b)), big.NewInt(0))
-
-	return c
-}
-
-// Big
-//
-// Shortcut for new(big.Int).SetString(..., 0)
-func Big(num string) *big.Int {
-	n := new(big.Int)
-	n.SetString(num, 0)
-
-	return n
-}
-
-// Bytes2Big
-//
-func BytesToBig(data []byte) *big.Int {
-	n := new(big.Int)
-	n.SetBytes(data)
-
-	return n
-}
-func Bytes2Big(data []byte) *big.Int { return BytesToBig(data) }
-func BigD(data []byte) *big.Int      { return BytesToBig(data) }
-
-func String2Big(num string) *big.Int {
-	n := new(big.Int)
-	n.SetString(num, 0)
-	return n
-}
-
-func BitTest(num *big.Int, i int) bool {
-	return num.Bit(i) > 0
-}
-
-// To256
-//
-// "cast" the big int to a 256 big int (i.e., limit to)
-var tt256 = new(big.Int).Lsh(big.NewInt(1), 256)
-var tt256m1 = new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1))
-var tt255 = new(big.Int).Lsh(big.NewInt(1), 255)
-
-func U256(x *big.Int) *big.Int {
-	//if x.Cmp(Big0) < 0 {
-	//		return new(big.Int).Add(tt256, x)
-	//	}
-
-	x.And(x, tt256m1)
-
-	return x
-}
-
-func S256(x *big.Int) *big.Int {
-	if x.Cmp(tt255) < 0 {
-		return x
-	} else {
-		// We don't want to modify x, ever
-		return new(big.Int).Sub(x, tt256)
-	}
-}
-
-func FirstBitSet(v *big.Int) int {
-	for i := 0; i < v.BitLen(); i++ {
-		if v.Bit(i) > 0 {
-			return i
-		}
-	}
-
-	return v.BitLen()
-}
-
-// Big to bytes
-//
-// Returns the bytes of a big integer with the size specified by **base**
-// Attempts to pad the byte array with zeros.
-func BigToBytes(num *big.Int, base int) []byte {
-	ret := make([]byte, base/8)
-
-	if len(num.Bytes()) > base/8 {
-		return num.Bytes()
-	}
-
-	return append(ret[:len(ret)-len(num.Bytes())], num.Bytes()...)
-}
-
-// Big copy
-//
-// Creates a copy of the given big integer
-func BigCopy(src *big.Int) *big.Int {
-	return new(big.Int).Set(src)
-}
-
-// Big max
-//
-// Returns the maximum size big integer
-func BigMax(x, y *big.Int) *big.Int {
-	if x.Cmp(y) < 0 {
-		return y
-	}
-
-	return x
-}
-
-// Big min
-//
-// Returns the minimum size big integer
-func BigMin(x, y *big.Int) *big.Int {
-	if x.Cmp(y) > 0 {
-		return y
-	}
-
-	return x
-}

common/big_test.go

@@ -1,89 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"bytes"
-	"testing"
-)
-
-func TestMisc(t *testing.T) {
-	a := Big("10")
-	b := Big("57896044618658097711785492504343953926634992332820282019728792003956564819968")
-	c := []byte{1, 2, 3, 4}
-	z := BitTest(a, 1)
-
-	if !z {
-		t.Error("Expected true got", z)
-	}
-
-	U256(a)
-	S256(a)
-
-	U256(b)
-	S256(b)
-
-	BigD(c)
-}
-
-func TestBigMax(t *testing.T) {
-	a := Big("10")
-	b := Big("5")
-
-	max1 := BigMax(a, b)
-	if max1 != a {
-		t.Errorf("Expected %d got %d", a, max1)
-	}
-
-	max2 := BigMax(b, a)
-	if max2 != a {
-		t.Errorf("Expected %d got %d", a, max2)
-	}
-}
-
-func TestBigMin(t *testing.T) {
-	a := Big("10")
-	b := Big("5")
-
-	min1 := BigMin(a, b)
-	if min1 != b {
-		t.Errorf("Expected %d got %d", b, min1)
-	}
-
-	min2 := BigMin(b, a)
-	if min2 != b {
-		t.Errorf("Expected %d got %d", b, min2)
-	}
-}
-
-func TestBigCopy(t *testing.T) {
-	a := Big("10")
-	b := BigCopy(a)
-	c := Big("1000000000000")
-	y := BigToBytes(b, 16)
-	ybytes := []byte{0, 10}
-	z := BigToBytes(c, 16)
-	zbytes := []byte{232, 212, 165, 16, 0}
-
-	if !bytes.Equal(y, ybytes) {
-		t.Error("Got", ybytes)
-	}
-
-	if !bytes.Equal(z, zbytes) {
-		t.Error("Got", zbytes)
-	}
-}

common/bitutil/bitutil.go

@@ -0,0 +1,188 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Adapted from: https://golang.org/src/crypto/cipher/xor.go
+
+// Package bitutil implements fast bitwise operations.
+package bitutil
+
+import (
+	"runtime"
+	"unsafe"
+)
+
+const wordSize = int(unsafe.Sizeof(uintptr(0)))
+const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x"
+
+// XORBytes xors the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes xor'd.
+func XORBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastXORBytes(dst, a, b)
+	}
+	return safeXORBytes(dst, a, b)
+}
+
+// fastXORBytes xors in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastXORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] ^ bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] ^ b[i]
+	}
+	return n
+}
+
+// safeXORBytes xors one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeXORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] ^ b[i]
+	}
+	return n
+}
+
+// ANDBytes ands the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes and'd.
+func ANDBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastANDBytes(dst, a, b)
+	}
+	return safeANDBytes(dst, a, b)
+}
+
+// fastANDBytes ands in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastANDBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] & bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] & b[i]
+	}
+	return n
+}
+
+// safeANDBytes ands one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeANDBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] & b[i]
+	}
+	return n
+}
+
+// ORBytes ors the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes or'd.
+func ORBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastORBytes(dst, a, b)
+	}
+	return safeORBytes(dst, a, b)
+}
+
+// fastORBytes ors in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] | bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] | b[i]
+	}
+	return n
+}
+
+// safeORBytes ors one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] | b[i]
+	}
+	return n
+}
+
+// TestBytes tests whether any bit is set in the input byte slice.
+func TestBytes(p []byte) bool {
+	if supportsUnaligned {
+		return fastTestBytes(p)
+	}
+	return safeTestBytes(p)
+}
+
+// fastTestBytes tests for set bits in bulk. It only works on architectures that
+// support unaligned read/writes.
+func fastTestBytes(p []byte) bool {
+	n := len(p)
+	w := n / wordSize
+	if w > 0 {
+		pw := *(*[]uintptr)(unsafe.Pointer(&p))
+		for i := 0; i < w; i++ {
+			if pw[i] != 0 {
+				return true
+			}
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		if p[i] != 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// safeTestBytes tests for set bits one byte at a time. It works on all
+// architectures, independent if it supports unaligned read/writes or not.
+func safeTestBytes(p []byte) bool {
+	for i := 0; i < len(p); i++ {
+		if p[i] != 0 {
+			return true
+		}
+	}
+	return false
+}

common/bitutil/bitutil_test.go

@@ -0,0 +1,215 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go
+
+package bitutil
+
+import (
+	"bytes"
+	"testing"
+)
+
+// Tests that bitwise XOR works for various alignments.
+func TestXOR(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				XORBytes(d1, p, q)
+				safeXORBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal", d1, d2)
+				}
+			}
+		}
+	}
+}
+
+// Tests that bitwise AND works for various alignments.
+func TestAND(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				ANDBytes(d1, p, q)
+				safeANDBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal")
+				}
+			}
+		}
+	}
+}
+
+// Tests that bitwise OR works for various alignments.
+func TestOR(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				ORBytes(d1, p, q)
+				safeORBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal")
+				}
+			}
+		}
+	}
+}
+
+// Tests that bit testing works for various alignments.
+func TestTest(t *testing.T) {
+	for align := 0; align < 2; align++ {
+		// Test for bits set in the bulk part
+		p := make([]byte, 1023)[align:]
+		p[100] = 1
+
+		if TestBytes(p) != safeTestBytes(p) {
+			t.Error("not equal")
+		}
+		// Test for bits set in the tail part
+		q := make([]byte, 1023)[align:]
+		q[len(q)-1] = 1
+
+		if TestBytes(q) != safeTestBytes(q) {
+			t.Error("not equal")
+		}
+	}
+}
+
+// Benchmarks the potentially optimized XOR performance.
+func BenchmarkFastXOR1KB(b *testing.B) { benchmarkFastXOR(b, 1024) }
+func BenchmarkFastXOR2KB(b *testing.B) { benchmarkFastXOR(b, 2048) }
+func BenchmarkFastXOR4KB(b *testing.B) { benchmarkFastXOR(b, 4096) }
+
+func benchmarkFastXOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		XORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline XOR performance.
+func BenchmarkBaseXOR1KB(b *testing.B) { benchmarkBaseXOR(b, 1024) }
+func BenchmarkBaseXOR2KB(b *testing.B) { benchmarkBaseXOR(b, 2048) }
+func BenchmarkBaseXOR4KB(b *testing.B) { benchmarkBaseXOR(b, 4096) }
+
+func benchmarkBaseXOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeXORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized AND performance.
+func BenchmarkFastAND1KB(b *testing.B) { benchmarkFastAND(b, 1024) }
+func BenchmarkFastAND2KB(b *testing.B) { benchmarkFastAND(b, 2048) }
+func BenchmarkFastAND4KB(b *testing.B) { benchmarkFastAND(b, 4096) }
+
+func benchmarkFastAND(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		ANDBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline AND performance.
+func BenchmarkBaseAND1KB(b *testing.B) { benchmarkBaseAND(b, 1024) }
+func BenchmarkBaseAND2KB(b *testing.B) { benchmarkBaseAND(b, 2048) }
+func BenchmarkBaseAND4KB(b *testing.B) { benchmarkBaseAND(b, 4096) }
+
+func benchmarkBaseAND(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeANDBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized OR performance.
+func BenchmarkFastOR1KB(b *testing.B) { benchmarkFastOR(b, 1024) }
+func BenchmarkFastOR2KB(b *testing.B) { benchmarkFastOR(b, 2048) }
+func BenchmarkFastOR4KB(b *testing.B) { benchmarkFastOR(b, 4096) }
+
+func benchmarkFastOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		ORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline OR performance.
+func BenchmarkBaseOR1KB(b *testing.B) { benchmarkBaseOR(b, 1024) }
+func BenchmarkBaseOR2KB(b *testing.B) { benchmarkBaseOR(b, 2048) }
+func BenchmarkBaseOR4KB(b *testing.B) { benchmarkBaseOR(b, 4096) }
+
+func benchmarkBaseOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized bit testing performance.
+func BenchmarkFastTest1KB(b *testing.B) { benchmarkFastTest(b, 1024) }
+func BenchmarkFastTest2KB(b *testing.B) { benchmarkFastTest(b, 2048) }
+func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) }
+
+func benchmarkFastTest(b *testing.B, size int) {
+	p := make([]byte, size)
+	for i := 0; i < b.N; i++ {
+		TestBytes(p)
+	}
+}
+
+// Benchmarks the baseline bit testing performance.
+func BenchmarkBaseTest1KB(b *testing.B) { benchmarkBaseTest(b, 1024) }
+func BenchmarkBaseTest2KB(b *testing.B) { benchmarkBaseTest(b, 2048) }
+func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) }
+
+func benchmarkBaseTest(b *testing.B, size int) {
+	p := make([]byte, size)
+	for i := 0; i < b.N; i++ {
+		safeTestBytes(p)
+	}
+}

common/bitutil/compress.go

@@ -0,0 +1,170 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bitutil
+
+import "errors"
+
+var (
+	// errMissingData is returned from decompression if the byte referenced by
+	// the bitset header overflows the input data.
+	errMissingData = errors.New("missing bytes on input")
+
+	// errUnreferencedData is returned from decompression if not all bytes were used
+	// up from the input data after decompressing it.
+	errUnreferencedData = errors.New("extra bytes on input")
+
+	// errExceededTarget is returned from decompression if the bitset header has
+	// more bits defined than the number of target buffer space available.
+	errExceededTarget = errors.New("target data size exceeded")
+
+	// errZeroContent is returned from decompression if a data byte referenced in
+	// the bitset header is actually a zero byte.
+	errZeroContent = errors.New("zero byte in input content")
+)
+
+// The compression algorithm implemented by CompressBytes and DecompressBytes is
+// optimized for sparse input data which contains a lot of zero bytes. Decompression
+// requires knowledge of the decompressed data length.
+//
+// Compression works as follows:
+//
+//   if data only contains zeroes,
+//       CompressBytes(data) == nil
+//   otherwise if len(data) <= 1,
+//       CompressBytes(data) == data
+//   otherwise:
+//       CompressBytes(data) == append(CompressBytes(nonZeroBitset(data)), nonZeroBytes(data)...)
+//       where
+//         nonZeroBitset(data) is a bit vector with len(data) bits (MSB first):
+//             nonZeroBitset(data)[i/8] && (1 << (7-i%8)) != 0  if data[i] != 0
+//             len(nonZeroBitset(data)) == (len(data)+7)/8
+//         nonZeroBytes(data) contains the non-zero bytes of data in the same order
+
+// CompressBytes compresses the input byte slice according to the sparse bitset
+// representation algorithm. If the result is bigger than the original input, no
+// compression is done.
+func CompressBytes(data []byte) []byte {
+	if out := bitsetEncodeBytes(data); len(out) < len(data) {
+		return out
+	}
+	cpy := make([]byte, len(data))
+	copy(cpy, data)
+	return cpy
+}
+
+// bitsetEncodeBytes compresses the input byte slice according to the sparse
+// bitset representation algorithm.
+func bitsetEncodeBytes(data []byte) []byte {
+	// Empty slices get compressed to nil
+	if len(data) == 0 {
+		return nil
+	}
+	// One byte slices compress to nil or retain the single byte
+	if len(data) == 1 {
+		if data[0] == 0 {
+			return nil
+		}
+		return data
+	}
+	// Calculate the bitset of set bytes, and gather the non-zero bytes
+	nonZeroBitset := make([]byte, (len(data)+7)/8)
+	nonZeroBytes := make([]byte, 0, len(data))
+
+	for i, b := range data {
+		if b != 0 {
+			nonZeroBytes = append(nonZeroBytes, b)
+			nonZeroBitset[i/8] |= 1 << byte(7-i%8)
+		}
+	}
+	if len(nonZeroBytes) == 0 {
+		return nil
+	}
+	return append(bitsetEncodeBytes(nonZeroBitset), nonZeroBytes...)
+}
+
+// DecompressBytes decompresses data with a known target size. If the input data
+// matches the size of the target, it means no compression was done in the first
+// place.
+func DecompressBytes(data []byte, target int) ([]byte, error) {
+	if len(data) > target {
+		return nil, errExceededTarget
+	}
+	if len(data) == target {
+		cpy := make([]byte, len(data))
+		copy(cpy, data)
+		return cpy, nil
+	}
+	return bitsetDecodeBytes(data, target)
+}
+
+// bitsetDecodeBytes decompresses data with a known target size.
+func bitsetDecodeBytes(data []byte, target int) ([]byte, error) {
+	out, size, err := bitsetDecodePartialBytes(data, target)
+	if err != nil {
+		return nil, err
+	}
+	if size != len(data) {
+		return nil, errUnreferencedData
+	}
+	return out, nil
+}
+
+// bitsetDecodePartialBytes decompresses data with a known target size, but does
+// not enforce consuming all the input bytes. In addition to the decompressed
+// output, the function returns the length of compressed input data corresponding
+// to the output as the input slice may be longer.
+func bitsetDecodePartialBytes(data []byte, target int) ([]byte, int, error) {
+	// Sanity check 0 targets to avoid infinite recursion
+	if target == 0 {
+		return nil, 0, nil
+	}
+	// Handle the zero and single byte corner cases
+	decomp := make([]byte, target)
+	if len(data) == 0 {
+		return decomp, 0, nil
+	}
+	if target == 1 {
+		decomp[0] = data[0] // copy to avoid referencing the input slice
+		if data[0] != 0 {
+			return decomp, 1, nil
+		}
+		return decomp, 0, nil
+	}
+	// Decompress the bitset of set bytes and distribute the non zero bytes
+	nonZeroBitset, ptr, err := bitsetDecodePartialBytes(data, (target+7)/8)
+	if err != nil {
+		return nil, ptr, err
+	}
+	for i := 0; i < 8*len(nonZeroBitset); i++ {
+		if nonZeroBitset[i/8]&(1<<byte(7-i%8)) != 0 {
+			// Make sure we have enough data to push into the correct slot
+			if ptr >= len(data) {
+				return nil, 0, errMissingData
+			}
+			if i >= len(decomp) {
+				return nil, 0, errExceededTarget
+			}
+			// Make sure the data is valid and push into the slot
+			if data[ptr] == 0 {
+				return nil, 0, errZeroContent
+			}
+			decomp[i] = data[ptr]
+			ptr++
+		}
+	}
+	return decomp, ptr, nil
+}

common/bitutil/compress_fuzz.go

@@ -0,0 +1,56 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build gofuzz
+
+package bitutil
+
+import "bytes"
+
+// Fuzz implements a go-fuzz fuzzer method to test various encoding method
+// invocations.
+func Fuzz(data []byte) int {
+	if len(data) == 0 {
+		return -1
+	}
+	if data[0]%2 == 0 {
+		return fuzzEncode(data[1:])
+	}
+	return fuzzDecode(data[1:])
+}
+
+// fuzzEncode implements a go-fuzz fuzzer method to test the bitset encoding and
+// decoding algorithm.
+func fuzzEncode(data []byte) int {
+	proc, _ := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data))
+	if !bytes.Equal(data, proc) {
+		panic("content mismatch")
+	}
+	return 0
+}
+
+// fuzzDecode implements a go-fuzz fuzzer method to test the bit decoding and
+// reencoding algorithm.
+func fuzzDecode(data []byte) int {
+	blob, err := bitsetDecodeBytes(data, 1024)
+	if err != nil {
+		return 0
+	}
+	if comp := bitsetEncodeBytes(blob); !bytes.Equal(comp, data) {
+		panic("content mismatch")
+	}
+	return 0
+}