params: release v1.7.3 stable

* eth, internal: Implement  using trie diffs

* eth, internal: Changes in response to review

* eth: More fixes to getModifiedAccountsBy*

* eth: minor polishes on error capitalization

.dockerignore

@@ -1,4 +1,7 @@
 **/.git
+.git
+!.git/HEAD
+!.git/refs/heads
 **/*_test.go
 
 build/_workspace

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+accounts/usbwallet @karalabe
+consensus          @karalabe
+core/              @karalabe @holiman
+eth/               @karalabe
+mobile/            @karalabe
+p2p/               @fjl @zsfelfoldi

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,9 @@
+Hi there,
+
+please note that this is an issue tracker reserved for bug reports and feature requests.
+
+For general questions please use the gitter channel or the Ethereum stack exchange at https://ethereum.stackexchange.com.
+
 #### System information
 
 Geth version: `geth version`

.gitignore

@@ -33,3 +33,8 @@ profile.cov
 
 # IdeaIDE
 .idea
+
+# dashboard
+/dashboard/assets/node_modules
+/dashboard/assets/stats.json
+/dashboard/assets/public/bundle.js

.travis.yml

@@ -38,7 +38,7 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage
 
     - os: osx
       go: 1.9.x
@@ -48,7 +48,21 @@ matrix:
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage
+
+    # This builder only tests code linters on latest version of Go
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.9.x
+      env:
+        - lint
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
@@ -133,16 +147,16 @@ matrix:
         - azure-android
         - maven-android
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -o android-ndk-r14b.zip
-        - unzip -q android-ndk-r14b.zip && rm android-ndk-r14b.zip
-        - mv android-ndk-r14b $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r14b
+        - curl https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -o android-ndk-r15c.zip
+        - unzip -q android-ndk-r15c.zip && rm android-ndk-r15c.zip
+        - mv android-ndk-r15c $HOME
+        - export ANDROID_NDK=$HOME/android-ndk-r15c
 
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum

Dockerfile

@@ -12,5 +12,5 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
-EXPOSE 8545 8546 30303 30303/udp
+EXPOSE 8545 8546 30303 30303/udp 30304/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -0,0 +1,15 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.9-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make all
+
+# Pull all binaries into a second stage deploy alpine container
+FROM alpine:latest
+
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
+
+EXPOSE 8545 8546 30303 30303/udp 30304/udp

README.md

@@ -35,7 +35,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
 | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
-| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
+| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
 | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
 | `swarm`    | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
@@ -130,6 +130,8 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
 
 This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does.  It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
 
+Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
+
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
@@ -264,7 +266,7 @@ instance for mining, run it with all your usual flags, extended by:
 $ geth <usual-flags> --mine --minerthreads=1 --etherbase=0x0000000000000000000000000000000000000000
 ```
 
-Which will start mining bocks and transactions on a single CPU thread, crediting all proceedings to
+Which will start mining blocks and transactions on a single CPU thread, crediting all proceedings to
 the account specified by `--etherbase`. You can further tune the mining by changing the default gas
 limit blocks converge to (`--targetgaslimit`) and the price transactions are accepted at (`--gasprice`).
 

VERSION

@@ -1 +1 @@
-1.7.1
+1.7.3

accounts/abi/abi.go

@@ -20,10 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"reflect"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
 )
 
 // The ABI holds information about a contract's context and available
@@ -76,106 +72,27 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
 	return append(method.Id(), arguments...), nil
 }
 
-// these variable are used to determine certain types during type assertion for
-// assignment.
-var (
-	r_interSlice = reflect.TypeOf([]interface{}{})
-	r_hash       = reflect.TypeOf(common.Hash{})
-	r_bytes      = reflect.TypeOf([]byte{})
-	r_byte       = reflect.TypeOf(byte(0))
-)
-
 // Unpack output in v according to the abi specification
-func (abi ABI) Unpack(v interface{}, name string, output []byte) error {
-	var method = abi.Methods[name]
-
-	if len(output) == 0 {
-		return fmt.Errorf("abi: unmarshalling empty output")
+func (abi ABI) Unpack(v interface{}, name string, output []byte) (err error) {
+	if err = bytesAreProper(output); err != nil {
+		return err
 	}
-
-	// make sure the passed value is a pointer
-	valueOf := reflect.ValueOf(v)
-	if reflect.Ptr != valueOf.Kind() {
-		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
-	}
-
-	var (
-		value = valueOf.Elem()
-		typ   = value.Type()
-	)
-
-	if len(method.Outputs) > 1 {
-		switch value.Kind() {
-		// struct will match named return values to the struct's field
-		// names
-		case reflect.Struct:
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				reflectValue := reflect.ValueOf(marshalledValue)
-
-				for j := 0; j < typ.NumField(); j++ {
-					field := typ.Field(j)
-					// TODO read tags: `abi:"fieldName"`
-					if field.Name == strings.ToUpper(method.Outputs[i].Name[:1])+method.Outputs[i].Name[1:] {
-						if err := set(value.Field(j), reflectValue, method.Outputs[i]); err != nil {
-							return err
-						}
-					}
-				}
-			}
-		case reflect.Slice:
-			if !value.Type().AssignableTo(r_interSlice) {
-				return fmt.Errorf("abi: cannot marshal tuple in to slice %T (only []interface{} is supported)", v)
-			}
-
-			// if the slice already contains values, set those instead of the interface slice itself.
-			if value.Len() > 0 {
-				if len(method.Outputs) > value.Len() {
-					return fmt.Errorf("abi: cannot marshal in to slices of unequal size (require: %v, got: %v)", len(method.Outputs), value.Len())
-				}
-
-				for i := 0; i < len(method.Outputs); i++ {
-					marshalledValue, err := toGoType(i, method.Outputs[i], output)
-					if err != nil {
-						return err
-					}
-					reflectValue := reflect.ValueOf(marshalledValue)
-					if err := set(value.Index(i).Elem(), reflectValue, method.Outputs[i]); err != nil {
-						return err
-					}
-				}
-				return nil
-			}
-
-			// create a new slice and start appending the unmarshalled
-			// values to the new interface slice.
-			z := reflect.MakeSlice(typ, 0, len(method.Outputs))
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				z = reflect.Append(z, reflect.ValueOf(marshalledValue))
-			}
-			value.Set(z)
-		default:
-			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
-		}
-
+	// since there can't be naming collisions with contracts and events,
+	// we need to decide whether we're calling a method or an event
+	var unpack unpacker
+	if method, ok := abi.Methods[name]; ok {
+		unpack = method
+	} else if event, ok := abi.Events[name]; ok {
+		unpack = event
 	} else {
-		marshalledValue, err := toGoType(0, method.Outputs[0], output)
-		if err != nil {
-			return err
-		}
-		if err := set(value, reflect.ValueOf(marshalledValue), method.Outputs[0]); err != nil {
-			return err
-		}
+		return fmt.Errorf("abi: could not locate named method or event.")
 	}
 
-	return nil
+	// requires a struct to unpack into for a tuple return...
+	if unpack.isTupleReturn() {
+		return unpack.tupleUnpack(v, output)
+	}
+	return unpack.singleUnpack(v, output)
 }
 
 func (abi *ABI) UnmarshalJSON(data []byte) error {

accounts/abi/abi_test.go

@@ -29,25 +29,6 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-// formatSilceOutput add padding to the value and adds a size
-func formatSliceOutput(v ...[]byte) []byte {
-	off := common.LeftPadBytes(big.NewInt(int64(len(v))).Bytes(), 32)
-	output := append(off, make([]byte, 0, len(v)*32)...)
-
-	for _, value := range v {
-		output = append(output, common.LeftPadBytes(value, 32)...)
-	}
-	return output
-}
-
-// quick helper padding
-func pad(input []byte, size int, left bool) []byte {
-	if left {
-		return common.LeftPadBytes(input, size)
-	}
-	return common.RightPadBytes(input, size)
-}
-
 const jsondata = `
 [
 	{ "type" : "function", "name" : "balance", "constant" : true },
@@ -191,7 +172,7 @@ func TestMethodSignature(t *testing.T) {
 		t.Errorf("expected ids to match %x != %x", m.Id(), idexp)
 	}
 
-	uintt, _ := NewType("uint")
+	uintt, _ := NewType("uint256")
 	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {

accounts/abi/bind/backends/simulated.go

@@ -41,6 +41,7 @@ import (
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
 var errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block")
+var errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction")
 
 // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in
 // the background. Its main purpose is to allow easily testing contract bindings.
@@ -59,7 +60,7 @@ type SimulatedBackend struct {
 // for testing purposes.
 func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
-	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
+	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
 	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
@@ -203,32 +204,46 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	// Binary search the gas requirement, as it may be higher than the amount used
+	// Determine the lowest and highest possible gas limits to binary search in between
 	var (
-		lo uint64 = params.TxGas - 1
-		hi uint64
+		lo  uint64 = params.TxGas - 1
+		hi  uint64
+		cap uint64
 	)
 	if call.Gas != nil && call.Gas.Uint64() >= params.TxGas {
 		hi = call.Gas.Uint64()
 	} else {
 		hi = b.pendingBlock.GasLimit().Uint64()
 	}
-	for lo+1 < hi {
-		// Take a guess at the gas, and check transaction validity
-		mid := (hi + lo) / 2
-		call.Gas = new(big.Int).SetUint64(mid)
+	cap = hi
+
+	// Create a helper to check if a gas allowance results in an executable transaction
+	executable := func(gas uint64) bool {
+		call.Gas = new(big.Int).SetUint64(gas)
 
 		snapshot := b.pendingState.Snapshot()
 		_, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
 		b.pendingState.RevertToSnapshot(snapshot)
 
-		// If the transaction became invalid or execution failed, raise the gas limit
 		if err != nil || failed {
-			lo = mid
-			continue
+			return false
+		}
+		return true
+	}
+	// Execute the binary search and hone in on an executable gas limit
+	for lo+1 < hi {
+		mid := (hi + lo) / 2
+		if !executable(mid) {
+			lo = mid
+		} else {
+			hi = mid
+		}
+	}
+	// Reject the transaction as invalid if it still fails at the highest allowance
+	if hi == cap {
+		if !executable(hi) {
+			return nil, errGasEstimationFailed
 		}
-		// Otherwise assume the transaction succeeded, lower the gas limit
-		hi = mid
 	}
 	return new(big.Int).SetUint64(hi), nil
 }

accounts/abi/error.go

@@ -39,22 +39,23 @@ func formatSliceString(kind reflect.Kind, sliceSize int) string {
 // type in t.
 func sliceTypeCheck(t Type, val reflect.Value) error {
 	if val.Kind() != reflect.Slice && val.Kind() != reflect.Array {
-		return typeErr(formatSliceString(t.Kind, t.SliceSize), val.Type())
-	}
-	if t.IsArray && val.Len() != t.SliceSize {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), formatSliceString(val.Type().Elem().Kind(), val.Len()))
+		return typeErr(formatSliceString(t.Kind, t.Size), val.Type())
 	}
 
-	if t.Elem.IsSlice {
+	if t.T == ArrayTy && val.Len() != t.Size {
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), formatSliceString(val.Type().Elem().Kind(), val.Len()))
+	}
+
+	if t.Elem.T == SliceTy {
 		if val.Len() > 0 {
 			return sliceTypeCheck(*t.Elem, val.Index(0))
 		}
-	} else if t.Elem.IsArray {
+	} else if t.Elem.T == ArrayTy {
 		return sliceTypeCheck(*t.Elem, val.Index(0))
 	}
 
 	if elemKind := val.Type().Elem().Kind(); elemKind != t.Elem.Kind {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), val.Type())
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), val.Type())
 	}
 	return nil
 }
@@ -62,20 +63,19 @@ func sliceTypeCheck(t Type, val reflect.Value) error {
 // typeCheck checks that the given reflection value can be assigned to the reflection
 // type in t.
 func typeCheck(t Type, value reflect.Value) error {
-	if t.IsSlice || t.IsArray {
+	if t.T == SliceTy || t.T == ArrayTy {
 		return sliceTypeCheck(t, value)
 	}
 
 	// Check base type validity. Element types will be checked later on.
 	if t.Kind != value.Kind() {
 		return typeErr(t.Kind, value.Kind())
+	} else if t.T == FixedBytesTy && t.Size != value.Len() {
+		return typeErr(t.Type, value.Type())
+	} else {
+		return nil
 	}
-	return nil
-}
 
-// varErr returns a formatted error.
-func varErr(expected, got reflect.Kind) error {
-	return typeErr(expected, got)
 }
 
 // typeErr returns a formatted type casting error.

accounts/abi/event.go

@@ -18,6 +18,7 @@ package abi
 
 import (
 	"fmt"
+	"reflect"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -44,3 +45,93 @@ func (e Event) Id() common.Hash {
 	}
 	return common.BytesToHash(crypto.Keccak256([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")))))
 }
+
+// unpacks an event return tuple into a struct of corresponding go types
+//
+// Unpacking can be done into a struct or a slice/array.
+func (e Event) tupleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	var (
+		value = valueOf.Elem()
+		typ   = value.Type()
+	)
+
+	if value.Kind() != reflect.Struct {
+		return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+	}
+
+	j := 0
+	for i := 0; i < len(e.Inputs); i++ {
+		input := e.Inputs[i]
+		if input.Indexed {
+			// can't read, continue
+			continue
+		} else if input.Type.T == ArrayTy {
+			// need to move this up because they read sequentially
+			j += input.Type.Size
+		}
+		marshalledValue, err := toGoType((i+j)*32, input.Type, output)
+		if err != nil {
+			return err
+		}
+		reflectValue := reflect.ValueOf(marshalledValue)
+
+		switch value.Kind() {
+		case reflect.Struct:
+			for j := 0; j < typ.NumField(); j++ {
+				field := typ.Field(j)
+				// TODO read tags: `abi:"fieldName"`
+				if field.Name == strings.ToUpper(e.Inputs[i].Name[:1])+e.Inputs[i].Name[1:] {
+					if err := set(value.Field(j), reflectValue, e.Inputs[i]); err != nil {
+						return err
+					}
+				}
+			}
+		case reflect.Slice, reflect.Array:
+			if value.Len() < i {
+				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(e.Inputs), value.Len())
+			}
+			v := value.Index(i)
+			if v.Kind() != reflect.Ptr && v.Kind() != reflect.Interface {
+				return fmt.Errorf("abi: cannot unmarshal %v in to %v", v.Type(), reflectValue.Type())
+			}
+			reflectValue := reflect.ValueOf(marshalledValue)
+			if err := set(v.Elem(), reflectValue, e.Inputs[i]); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+		}
+	}
+	return nil
+}
+
+func (e Event) isTupleReturn() bool { return len(e.Inputs) > 1 }
+
+func (e Event) singleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	if e.Inputs[0].Indexed {
+		return fmt.Errorf("abi: attempting to unpack indexed variable into element.")
+	}
+
+	value := valueOf.Elem()
+
+	marshalledValue, err := toGoType(0, e.Inputs[0].Type, output)
+	if err != nil {
+		return err
+	}
+	if err := set(value, reflect.ValueOf(marshalledValue), e.Inputs[0]); err != nil {
+		return err
+	}
+	return nil
+}

accounts/abi/event_test.go

@@ -31,7 +31,7 @@ func TestEventId(t *testing.T) {
 	}{
 		{
 			definition: `[
-			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint" }] },
+			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
 			{ "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }
 			]`,
 			expectations: map[string]common.Hash{

accounts/abi/method.go

@@ -77,6 +77,85 @@ func (method Method) pack(args ...interface{}) ([]byte, error) {
 	return ret, nil
 }
 
+// unpacks a method return tuple into a struct of corresponding go types
+//
+// Unpacking can be done into a struct or a slice/array.
+func (method Method) tupleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	var (
+		value = valueOf.Elem()
+		typ   = value.Type()
+	)
+
+	j := 0
+	for i := 0; i < len(method.Outputs); i++ {
+		toUnpack := method.Outputs[i]
+		if toUnpack.Type.T == ArrayTy {
+			// need to move this up because they read sequentially
+			j += toUnpack.Type.Size
+		}
+		marshalledValue, err := toGoType((i+j)*32, toUnpack.Type, output)
+		if err != nil {
+			return err
+		}
+		reflectValue := reflect.ValueOf(marshalledValue)
+
+		switch value.Kind() {
+		case reflect.Struct:
+			for j := 0; j < typ.NumField(); j++ {
+				field := typ.Field(j)
+				// TODO read tags: `abi:"fieldName"`
+				if field.Name == strings.ToUpper(method.Outputs[i].Name[:1])+method.Outputs[i].Name[1:] {
+					if err := set(value.Field(j), reflectValue, method.Outputs[i]); err != nil {
+						return err
+					}
+				}
+			}
+		case reflect.Slice, reflect.Array:
+			if value.Len() < i {
+				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(method.Outputs), value.Len())
+			}
+			v := value.Index(i)
+			if v.Kind() != reflect.Ptr && v.Kind() != reflect.Interface {
+				return fmt.Errorf("abi: cannot unmarshal %v in to %v", v.Type(), reflectValue.Type())
+			}
+			reflectValue := reflect.ValueOf(marshalledValue)
+			if err := set(v.Elem(), reflectValue, method.Outputs[i]); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+		}
+	}
+	return nil
+}
+
+func (method Method) isTupleReturn() bool { return len(method.Outputs) > 1 }
+
+func (method Method) singleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	value := valueOf.Elem()
+
+	marshalledValue, err := toGoType(0, method.Outputs[0].Type, output)
+	if err != nil {
+		return err
+	}
+	if err := set(value, reflect.ValueOf(marshalledValue), method.Outputs[0]); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Sig returns the methods string signature according to the ABI spec.
 //
 // Example

accounts/abi/numbers.go

@@ -25,36 +25,23 @@ import (
 )
 
 var (
-	big_t     = reflect.TypeOf(big.Int{})
-	ubig_t    = reflect.TypeOf(big.Int{})
-	byte_t    = reflect.TypeOf(byte(0))
-	byte_ts   = reflect.TypeOf([]byte(nil))
-	uint_t    = reflect.TypeOf(uint(0))
-	uint8_t   = reflect.TypeOf(uint8(0))
-	uint16_t  = reflect.TypeOf(uint16(0))
-	uint32_t  = reflect.TypeOf(uint32(0))
-	uint64_t  = reflect.TypeOf(uint64(0))
-	int_t     = reflect.TypeOf(int(0))
-	int8_t    = reflect.TypeOf(int8(0))
-	int16_t   = reflect.TypeOf(int16(0))
-	int32_t   = reflect.TypeOf(int32(0))
-	int64_t   = reflect.TypeOf(int64(0))
-	hash_t    = reflect.TypeOf(common.Hash{})
-	address_t = reflect.TypeOf(common.Address{})
-
-	uint_ts   = reflect.TypeOf([]uint(nil))
-	uint8_ts  = reflect.TypeOf([]uint8(nil))
-	uint16_ts = reflect.TypeOf([]uint16(nil))
-	uint32_ts = reflect.TypeOf([]uint32(nil))
-	uint64_ts = reflect.TypeOf([]uint64(nil))
-	ubig_ts   = reflect.TypeOf([]*big.Int(nil))
-
-	int_ts   = reflect.TypeOf([]int(nil))
-	int8_ts  = reflect.TypeOf([]int8(nil))
-	int16_ts = reflect.TypeOf([]int16(nil))
-	int32_ts = reflect.TypeOf([]int32(nil))
-	int64_ts = reflect.TypeOf([]int64(nil))
-	big_ts   = reflect.TypeOf([]*big.Int(nil))
+	big_t      = reflect.TypeOf(&big.Int{})
+	derefbig_t = reflect.TypeOf(big.Int{})
+	uint8_t    = reflect.TypeOf(uint8(0))
+	uint16_t   = reflect.TypeOf(uint16(0))
+	uint32_t   = reflect.TypeOf(uint32(0))
+	uint64_t   = reflect.TypeOf(uint64(0))
+	int_t      = reflect.TypeOf(int(0))
+	int8_t     = reflect.TypeOf(int8(0))
+	int16_t    = reflect.TypeOf(int16(0))
+	int32_t    = reflect.TypeOf(int32(0))
+	int64_t    = reflect.TypeOf(int64(0))
+	address_t  = reflect.TypeOf(common.Address{})
+	int_ts     = reflect.TypeOf([]int(nil))
+	int8_ts    = reflect.TypeOf([]int8(nil))
+	int16_ts   = reflect.TypeOf([]int16(nil))
+	int32_ts   = reflect.TypeOf([]int32(nil))
+	int64_ts   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.

accounts/abi/pack.go

@@ -61,8 +61,9 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 			reflectValue = mustArrayToByteSlice(reflectValue)
 		}
 		return common.RightPadBytes(reflectValue.Bytes(), 32)
+	default:
+		panic("abi: fatal error")
 	}
-	panic("abi: fatal error")
 }
 
 // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation
@@ -74,6 +75,8 @@ func packNum(value reflect.Value) []byte {
 		return U256(big.NewInt(value.Int()))
 	case reflect.Ptr:
 		return U256(value.Interface().(*big.Int))
+	default:
+		panic("abi: fatal error")
 	}
-	return nil
+
 }

accounts/abi/pack_test.go

@@ -322,12 +322,12 @@ func TestPack(t *testing.T) {
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
-			t.Fatal("unexpected parse error:", err)
+			t.Fatalf("%v failed. Unexpected parse error: %v", i, err)
 		}
 
 		output, err := typ.pack(reflect.ValueOf(test.input))
 		if err != nil {
-			t.Fatal("unexpected pack error:", err)
+			t.Fatalf("%v failed. Unexpected pack error: %v", i, err)
 		}
 
 		if !bytes.Equal(output, test.output) {
@@ -435,7 +435,4 @@ func TestPackNumber(t *testing.T) {
 			t.Errorf("test %d: pack mismatch: have %x, want %x", i, packed, tt.packed)
 		}
 	}
-	if packed := packNum(reflect.ValueOf("string")); packed != nil {
-		t.Errorf("expected 'string' to pack to nil. got %x instead", packed)
-	}
 }

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != big_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
 		return indirect(v.Elem())
 	}
 	return v
@@ -73,15 +73,9 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 func set(dst, src reflect.Value, output Argument) error {
 	dstType := dst.Type()
 	srcType := src.Type()
-
 	switch {
-	case dstType.AssignableTo(src.Type()):
+	case dstType.AssignableTo(srcType):
 		dst.Set(src)
-	case dstType.Kind() == reflect.Array && srcType.Kind() == reflect.Slice:
-		if dst.Len() < output.Type.SliceSize {
-			return fmt.Errorf("abi: cannot unmarshal src (len=%d) in to dst (len=%d)", output.Type.SliceSize, dst.Len())
-		}
-		reflect.Copy(dst, src)
 	case dstType.Kind() == reflect.Interface:
 		dst.Set(src)
 	case dstType.Kind() == reflect.Ptr:

accounts/abi/type.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 )
 
 const (
@@ -29,6 +30,7 @@ const (
 	BoolTy
 	StringTy
 	SliceTy
+	ArrayTy
 	AddressTy
 	FixedBytesTy
 	BytesTy
@@ -39,9 +41,6 @@ const (
 
 // Type is the reflection of the supported argument type
 type Type struct {
-	IsSlice, IsArray bool
-	SliceSize        int
-
 	Elem *Type
 
 	Kind reflect.Kind
@@ -53,118 +52,116 @@ type Type struct {
 }
 
 var (
-	// fullTypeRegex parses the abi types
-	//
-	// Types can be in the format of:
-	//
-	// 	Input  = Type [ "[" [ Number ] "]" ] Name .
-	// 	Type   = [ "u" ] "int" [ Number ] [ x ] [ Number ].
-	//
-	// Examples:
-	//
-	//      string     int       uint       fixed
-	//      string32   int8      uint8      uint[]
-	//      address    int256    uint256    fixed128x128[2]
-	fullTypeRegex = regexp.MustCompile(`([a-zA-Z0-9]+)(\[([0-9]*)\])?`)
 	// typeRegex parses the abi sub types
 	typeRegex = regexp.MustCompile("([a-zA-Z]+)(([0-9]+)(x([0-9]+))?)?")
 )
 
 // NewType creates a new reflection type of abi type given in t.
 func NewType(t string) (typ Type, err error) {
-	res := fullTypeRegex.FindAllStringSubmatch(t, -1)[0]
-	// check if type is slice and parse type.
-	switch {
-	case res[3] != "":
-		// err is ignored. Already checked for number through the regexp
-		typ.SliceSize, _ = strconv.Atoi(res[3])
-		typ.IsArray = true
-	case res[2] != "":
-		typ.IsSlice, typ.SliceSize = true, -1
-	case res[0] == "":
-		return Type{}, fmt.Errorf("abi: type parse error: %s", t)
+	// check that array brackets are equal if they exist
+	if strings.Count(t, "[") != strings.Count(t, "]") {
+		return Type{}, fmt.Errorf("invalid arg type in abi")
 	}
-	if typ.IsArray || typ.IsSlice {
-		sliceType, err := NewType(res[1])
+
+	typ.stringKind = t
+
+	// if there are brackets, get ready to go into slice/array mode and
+	// recursively create the type
+	if strings.Count(t, "[") != 0 {
+		i := strings.LastIndex(t, "[")
+		// recursively embed the type
+		embeddedType, err := NewType(t[:i])
 		if err != nil {
 			return Type{}, err
 		}
-		typ.Elem = &sliceType
-		typ.stringKind = sliceType.stringKind + t[len(res[1]):]
-		// Although we know that this is an array, we cannot return
-		// as we don't know the type of the element, however, if it
-		// is still an array, then don't determine the type.
-		if typ.Elem.IsArray || typ.Elem.IsSlice {
-			return typ, nil
-		}
-	}
+		// grab the last cell and create a type from there
+		sliced := t[i:]
+		// grab the slice size with regexp
+		re := regexp.MustCompile("[0-9]+")
+		intz := re.FindAllString(sliced, -1)
 
-	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(res[1], -1)[0]
-	// varSize is the size of the variable
-	var varSize int
-	if len(parsedType[3]) > 0 {
-		var err error
-		varSize, err = strconv.Atoi(parsedType[2])
-		if err != nil {
-			return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
-		}
-	}
-	// varType is the parsed abi type
-	varType := parsedType[1]
-	// substitute canonical integer
-	if varSize == 0 && (varType == "int" || varType == "uint") {
-		varSize = 256
-		t += "256"
-	}
-
-	// only set stringKind if not array or slice, as for those,
-	// the correct string type has been set
-	if !(typ.IsArray || typ.IsSlice) {
-		typ.stringKind = t
-	}
-
-	switch varType {
-	case "int":
-		typ.Kind, typ.Type = reflectIntKindAndType(false, varSize)
-		typ.Size = varSize
-		typ.T = IntTy
-	case "uint":
-		typ.Kind, typ.Type = reflectIntKindAndType(true, varSize)
-		typ.Size = varSize
-		typ.T = UintTy
-	case "bool":
-		typ.Kind = reflect.Bool
-		typ.T = BoolTy
-	case "address":
-		typ.Kind = reflect.Array
-		typ.Type = address_t
-		typ.Size = 20
-		typ.T = AddressTy
-	case "string":
-		typ.Kind = reflect.String
-		typ.Size = -1
-		typ.T = StringTy
-	case "bytes":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
-		if varSize == 0 {
-			typ.IsSlice = true
-			typ.T = BytesTy
-			typ.SliceSize = -1
+		if len(intz) == 0 {
+			// is a slice
+			typ.T = SliceTy
+			typ.Kind = reflect.Slice
+			typ.Elem = &embeddedType
+			typ.Type = reflect.SliceOf(embeddedType.Type)
+		} else if len(intz) == 1 {
+			// is a array
+			typ.T = ArrayTy
+			typ.Kind = reflect.Array
+			typ.Elem = &embeddedType
+			typ.Size, err = strconv.Atoi(intz[0])
+			if err != nil {
+				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
+			}
+			typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type)
 		} else {
-			typ.IsArray = true
-			typ.T = FixedBytesTy
-			typ.SliceSize = varSize
+			return Type{}, fmt.Errorf("invalid formatting of array type")
+		}
+		return typ, err
+	} else {
+		// parse the type and size of the abi-type.
+		parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+		// varSize is the size of the variable
+		var varSize int
+		if len(parsedType[3]) > 0 {
+			var err error
+			varSize, err = strconv.Atoi(parsedType[2])
+			if err != nil {
+				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
+			}
+		} else {
+			if parsedType[0] == "uint" || parsedType[0] == "int" {
+				// this should fail because it means that there's something wrong with
+				// the abi type (the compiler should always format it to the size...always)
+				return Type{}, fmt.Errorf("unsupported arg type: %s", t)
+			}
+		}
+		// varType is the parsed abi type
+		varType := parsedType[1]
+
+		switch varType {
+		case "int":
+			typ.Kind, typ.Type = reflectIntKindAndType(false, varSize)
+			typ.Size = varSize
+			typ.T = IntTy
+		case "uint":
+			typ.Kind, typ.Type = reflectIntKindAndType(true, varSize)
+			typ.Size = varSize
+			typ.T = UintTy
+		case "bool":
+			typ.Kind = reflect.Bool
+			typ.T = BoolTy
+			typ.Type = reflect.TypeOf(bool(false))
+		case "address":
+			typ.Kind = reflect.Array
+			typ.Type = address_t
+			typ.Size = 20
+			typ.T = AddressTy
+		case "string":
+			typ.Kind = reflect.String
+			typ.Type = reflect.TypeOf("")
+			typ.T = StringTy
+		case "bytes":
+			if varSize == 0 {
+				typ.T = BytesTy
+				typ.Kind = reflect.Slice
+				typ.Type = reflect.SliceOf(reflect.TypeOf(byte(0)))
+			} else {
+				typ.T = FixedBytesTy
+				typ.Kind = reflect.Array
+				typ.Size = varSize
+				typ.Type = reflect.ArrayOf(varSize, reflect.TypeOf(byte(0)))
+			}
+		case "function":
+			typ.Kind = reflect.Array
+			typ.T = FunctionTy
+			typ.Size = 24
+			typ.Type = reflect.ArrayOf(24, reflect.TypeOf(byte(0)))
+		default:
+			return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 		}
-	case "function":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
-		typ.IsArray = true
-		typ.T = FunctionTy
-		typ.SliceSize = 24
-	default:
-		return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 	}
 
 	return
@@ -183,7 +180,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if (t.IsSlice || t.IsArray) && t.T != BytesTy && t.T != FixedBytesTy && t.T != FunctionTy {
+	if t.T == SliceTy || t.T == ArrayTy {
 		var packed []byte
 
 		for i := 0; i < v.Len(); i++ {
@@ -193,18 +190,17 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 			}
 			packed = append(packed, val...)
 		}
-		if t.IsSlice {
+		if t.T == SliceTy {
 			return packBytesSlice(packed, v.Len()), nil
-		} else if t.IsArray {
+		} else if t.T == ArrayTy {
 			return packed, nil
 		}
 	}
-
 	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
 // prefixing.
 func (t Type) requiresLengthPrefix() bool {
-	return t.T != FixedBytesTy && (t.T == StringTy || t.T == BytesTy || t.IsSlice)
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }

accounts/abi/type_test.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
 )
 
@@ -34,51 +35,58 @@ func TestTypeRegexp(t *testing.T) {
 		blob string
 		kind Type
 	}{
-		{"bool", Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}},
-		{"bool[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[]"}},
-		{"bool[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool", Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}},
+		{"bool[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}},
+		{"bool[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool[2][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}},
+		{"bool[][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}},
+		{"bool[][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}},
+		{"bool[2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}},
+		{"bool[2][][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}},
+		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
+		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
+		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
 		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
 		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
 		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
 		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
 		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
 		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
 		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
 		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
 		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
 		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
-		{"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}},
-		{"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
-		{"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}},
-		{"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
-		{"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
-		{"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}},
-		{"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}},
-		{"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
+		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
+		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
+		{"bytes32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
+		{"bytes32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
+		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
+		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
+		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
 		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
-
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -87,13 +95,14 @@ func TestTypeRegexp(t *testing.T) {
 		// {"fixed128x128[]", Type{}},
 		// {"fixed128x128[2]", Type{}},
 	}
-	for i, tt := range tests {
+
+	for _, tt := range tests {
 		typ, err := NewType(tt.blob)
 		if err != nil {
-			t.Errorf("type %d: failed to parse type string: %v", i, err)
+			t.Errorf("type %q: failed to parse type string: %v", tt.blob, err)
 		}
 		if !reflect.DeepEqual(typ, tt.kind) {
-			t.Errorf("type %d: parsed type mismatch:\n  have %+v\n  want %+v", i, typeWithoutStringer(typ), typeWithoutStringer(tt.kind))
+			t.Errorf("type %q: parsed type mismatch:\nGOT %s\nWANT %s ", tt.blob, spew.Sdump(typeWithoutStringer(typ)), spew.Sdump(typeWithoutStringer(tt.kind)))
 		}
 	}
 }
@@ -104,15 +113,90 @@ func TestTypeCheck(t *testing.T) {
 		input interface{}
 		err   string
 	}{
-		{"uint", big.NewInt(1), ""},
-		{"int", big.NewInt(1), ""},
-		{"uint30", big.NewInt(1), ""},
+		{"uint", big.NewInt(1), "unsupported arg type: uint"},
+		{"int", big.NewInt(1), "unsupported arg type: int"},
+		{"uint256", big.NewInt(1), ""},
+		{"uint256[][3][]", [][3][]*big.Int{{{}}}, ""},
+		{"uint256[][][3]", [3][][]*big.Int{{{}}}, ""},
+		{"uint256[3][][]", [][][3]*big.Int{{{}}}, ""},
+		{"uint256[3][3][3]", [3][3][3]*big.Int{{{}}}, ""},
+		{"uint8[][]", [][]uint8{}, ""},
+		{"int256", big.NewInt(1), ""},
+		{"uint8", uint8(1), ""},
+		{"uint16", uint16(1), ""},
+		{"uint32", uint32(1), ""},
+		{"uint64", uint64(1), ""},
+		{"int8", int8(1), ""},
+		{"int16", int16(1), ""},
+		{"int32", int32(1), ""},
+		{"int64", int64(1), ""},
+		{"uint24", big.NewInt(1), ""},
+		{"uint40", big.NewInt(1), ""},
+		{"uint48", big.NewInt(1), ""},
+		{"uint56", big.NewInt(1), ""},
+		{"uint72", big.NewInt(1), ""},
+		{"uint80", big.NewInt(1), ""},
+		{"uint88", big.NewInt(1), ""},
+		{"uint96", big.NewInt(1), ""},
+		{"uint104", big.NewInt(1), ""},
+		{"uint112", big.NewInt(1), ""},
+		{"uint120", big.NewInt(1), ""},
+		{"uint128", big.NewInt(1), ""},
+		{"uint136", big.NewInt(1), ""},
+		{"uint144", big.NewInt(1), ""},
+		{"uint152", big.NewInt(1), ""},
+		{"uint160", big.NewInt(1), ""},
+		{"uint168", big.NewInt(1), ""},
+		{"uint176", big.NewInt(1), ""},
+		{"uint184", big.NewInt(1), ""},
+		{"uint192", big.NewInt(1), ""},
+		{"uint200", big.NewInt(1), ""},
+		{"uint208", big.NewInt(1), ""},
+		{"uint216", big.NewInt(1), ""},
+		{"uint224", big.NewInt(1), ""},
+		{"uint232", big.NewInt(1), ""},
+		{"uint240", big.NewInt(1), ""},
+		{"uint248", big.NewInt(1), ""},
+		{"int24", big.NewInt(1), ""},
+		{"int40", big.NewInt(1), ""},
+		{"int48", big.NewInt(1), ""},
+		{"int56", big.NewInt(1), ""},
+		{"int72", big.NewInt(1), ""},
+		{"int80", big.NewInt(1), ""},
+		{"int88", big.NewInt(1), ""},
+		{"int96", big.NewInt(1), ""},
+		{"int104", big.NewInt(1), ""},
+		{"int112", big.NewInt(1), ""},
+		{"int120", big.NewInt(1), ""},
+		{"int128", big.NewInt(1), ""},
+		{"int136", big.NewInt(1), ""},
+		{"int144", big.NewInt(1), ""},
+		{"int152", big.NewInt(1), ""},
+		{"int160", big.NewInt(1), ""},
+		{"int168", big.NewInt(1), ""},
+		{"int176", big.NewInt(1), ""},
+		{"int184", big.NewInt(1), ""},
+		{"int192", big.NewInt(1), ""},
+		{"int200", big.NewInt(1), ""},
+		{"int208", big.NewInt(1), ""},
+		{"int216", big.NewInt(1), ""},
+		{"int224", big.NewInt(1), ""},
+		{"int232", big.NewInt(1), ""},
+		{"int240", big.NewInt(1), ""},
+		{"int248", big.NewInt(1), ""},
 		{"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"},
+		{"uint8", uint16(1), "abi: cannot use uint16 as type uint8 as argument"},
+		{"uint8", uint32(1), "abi: cannot use uint32 as type uint8 as argument"},
+		{"uint8", uint64(1), "abi: cannot use uint64 as type uint8 as argument"},
+		{"uint8", int8(1), "abi: cannot use int8 as type uint8 as argument"},
+		{"uint8", int16(1), "abi: cannot use int16 as type uint8 as argument"},
+		{"uint8", int32(1), "abi: cannot use int32 as type uint8 as argument"},
+		{"uint8", int64(1), "abi: cannot use int64 as type uint8 as argument"},
 		{"uint16", uint16(1), ""},
 		{"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"},
 		{"uint16[]", []uint16{1, 2, 3}, ""},
 		{"uint16[]", [3]uint16{1, 2, 3}, ""},
-		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type []uint16 as argument"},
+		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"},
 		{"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"},
 		{"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
 		{"uint16[3]", []uint16{1, 2, 3}, ""},
@@ -122,20 +206,61 @@ func TestTypeCheck(t *testing.T) {
 		{"address[1]", [1]common.Address{{1}}, ""},
 		{"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
 		{"bytes32", [32]byte{}, ""},
+		{"bytes31", [31]byte{}, ""},
+		{"bytes30", [30]byte{}, ""},
+		{"bytes29", [29]byte{}, ""},
+		{"bytes28", [28]byte{}, ""},
+		{"bytes27", [27]byte{}, ""},
+		{"bytes26", [26]byte{}, ""},
+		{"bytes25", [25]byte{}, ""},
+		{"bytes24", [24]byte{}, ""},
+		{"bytes23", [23]byte{}, ""},
+		{"bytes22", [22]byte{}, ""},
+		{"bytes21", [21]byte{}, ""},
+		{"bytes20", [20]byte{}, ""},
+		{"bytes19", [19]byte{}, ""},
+		{"bytes18", [18]byte{}, ""},
+		{"bytes17", [17]byte{}, ""},
+		{"bytes16", [16]byte{}, ""},
+		{"bytes15", [15]byte{}, ""},
+		{"bytes14", [14]byte{}, ""},
+		{"bytes13", [13]byte{}, ""},
+		{"bytes12", [12]byte{}, ""},
+		{"bytes11", [11]byte{}, ""},
+		{"bytes10", [10]byte{}, ""},
+		{"bytes9", [9]byte{}, ""},
+		{"bytes8", [8]byte{}, ""},
+		{"bytes7", [7]byte{}, ""},
+		{"bytes6", [6]byte{}, ""},
+		{"bytes5", [5]byte{}, ""},
+		{"bytes4", [4]byte{}, ""},
+		{"bytes3", [3]byte{}, ""},
+		{"bytes2", [2]byte{}, ""},
+		{"bytes1", [1]byte{}, ""},
 		{"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
 		{"bytes32", common.Hash{1}, ""},
-		{"bytes31", [31]byte{}, ""},
+		{"bytes31", common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"},
 		{"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"},
 		{"bytes", []byte{0, 1}, ""},
-		{"bytes", [2]byte{0, 1}, ""},
-		{"bytes", common.Hash{1}, ""},
+		{"bytes", [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
+		{"bytes", common.Hash{1}, "abi: cannot use array as type slice as argument"},
 		{"string", "hello world", ""},
+		{"string", string(""), ""},
+		{"string", []byte{}, "abi: cannot use slice as type string as argument"},
 		{"bytes32[]", [][32]byte{{}}, ""},
 		{"function", [24]byte{}, ""},
+		{"bytes20", common.Address{}, ""},
+		{"address", [20]byte{}, ""},
+		{"address", common.Address{}, ""},
 	} {
 		typ, err := NewType(test.typ)
-		if err != nil {
+		if err != nil && len(test.err) == 0 {
 			t.Fatal("unexpected parse error:", err)
+		} else if err != nil && len(test.err) != 0 {
+			if err.Error() != test.err {
+				t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err)
+			}
+			continue
 		}
 
 		err = typeCheck(typ, reflect.ValueOf(test.input))

accounts/abi/unpack.go

@@ -25,122 +25,20 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-// toGoSliceType parses the input and casts it to the proper slice defined by the ABI
-// argument in T.
-func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
-	index := i * 32
-	// The slice must, at very least be large enough for the index+32 which is exactly the size required
-	// for the [offset in output, size of offset].
-	if index+32 > len(output) {
-		return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), index+32)
-	}
-	elem := t.Type.Elem
-
-	// first we need to create a slice of the type
-	var refSlice reflect.Value
-	switch elem.T {
-	case IntTy, UintTy, BoolTy:
-		// create a new reference slice matching the element type
-		switch t.Type.Kind {
-		case reflect.Bool:
-			refSlice = reflect.ValueOf([]bool(nil))
-		case reflect.Uint8:
-			refSlice = reflect.ValueOf([]uint8(nil))
-		case reflect.Uint16:
-			refSlice = reflect.ValueOf([]uint16(nil))
-		case reflect.Uint32:
-			refSlice = reflect.ValueOf([]uint32(nil))
-		case reflect.Uint64:
-			refSlice = reflect.ValueOf([]uint64(nil))
-		case reflect.Int8:
-			refSlice = reflect.ValueOf([]int8(nil))
-		case reflect.Int16:
-			refSlice = reflect.ValueOf([]int16(nil))
-		case reflect.Int32:
-			refSlice = reflect.ValueOf([]int32(nil))
-		case reflect.Int64:
-			refSlice = reflect.ValueOf([]int64(nil))
-		default:
-			refSlice = reflect.ValueOf([]*big.Int(nil))
-		}
-	case AddressTy: // address must be of slice Address
-		refSlice = reflect.ValueOf([]common.Address(nil))
-	case HashTy: // hash must be of slice hash
-		refSlice = reflect.ValueOf([]common.Hash(nil))
-	case FixedBytesTy:
-		refSlice = reflect.ValueOf([][]byte(nil))
-	default: // no other types are supported
-		return nil, fmt.Errorf("abi: unsupported slice type %v", elem.T)
-	}
-
-	var slice []byte
-	var size int
-	var offset int
-	if t.Type.IsSlice {
-		// get the offset which determines the start of this array ...
-		offset = int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
-		}
-
-		slice = output[offset:]
-		// ... starting with the size of the array in elements ...
-		size = int(binary.BigEndian.Uint64(slice[24:32]))
-		slice = slice[32:]
-		// ... and make sure that we've at the very least the amount of bytes
-		// available in the buffer.
-		if size*32 > len(slice) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), offset+32+size*32)
-		}
-
-		// reslice to match the required size
-		slice = slice[:size*32]
-	} else if t.Type.IsArray {
-		//get the number of elements in the array
-		size = t.Type.SliceSize
-
-		//check to make sure array size matches up
-		if index+32*size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), index+32*size)
-		}
-		//slice is there for a fixed amount of times
-		slice = output[index : index+size*32]
-	}
-
-	for i := 0; i < size; i++ {
-		var (
-			inter        interface{}             // interface type
-			returnOutput = slice[i*32 : i*32+32] // the return output
-			err          error
-		)
-		// set inter to the correct type (cast)
-		switch elem.T {
-		case IntTy, UintTy:
-			inter = readInteger(t.Type.Kind, returnOutput)
-		case BoolTy:
-			inter, err = readBool(returnOutput)
-			if err != nil {
-				return nil, err
-			}
-		case AddressTy:
-			inter = common.BytesToAddress(returnOutput)
-		case HashTy:
-			inter = common.BytesToHash(returnOutput)
-		case FixedBytesTy:
-			inter = returnOutput
-		}
-		// append the item to our reflect slice
-		refSlice = reflect.Append(refSlice, reflect.ValueOf(inter))
-	}
-
-	// return the interface
-	return refSlice.Interface(), nil
+// unpacker is a utility interface that enables us to have
+// abstraction between events and methods and also to properly
+// "unpack" them; e.g. events use Inputs, methods use Outputs.
+type unpacker interface {
+	tupleUnpack(v interface{}, output []byte) error
+	singleUnpack(v interface{}, output []byte) error
+	isTupleReturn() bool
 }
 
+// reads the integer based on its kind
 func readInteger(kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
-		return uint8(b[len(b)-1])
+		return b[len(b)-1]
 	case reflect.Uint16:
 		return binary.BigEndian.Uint16(b[len(b)-2:])
 	case reflect.Uint32:
@@ -160,13 +58,10 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	}
 }
 
+// reads a bool
 func readBool(word []byte) (bool, error) {
-	if len(word) != 32 {
-		return false, fmt.Errorf("abi: fatal error: incorrect word length")
-	}
-
-	for i, b := range word {
-		if b != 0 && i != 31 {
+	for _, b := range word[:31] {
+		if b != 0 {
 			return false, errBadBool
 		}
 	}
@@ -178,58 +73,144 @@ func readBool(word []byte) (bool, error) {
 	default:
 		return false, errBadBool
 	}
+}
+
+// A function type is simply the address with the function selection signature at the end.
+// This enforces that standard by always presenting it as a 24-array (address + sig = 24 bytes)
+func readFunctionType(t Type, word []byte) (funcTy [24]byte, err error) {
+	if t.T != FunctionTy {
+		return [24]byte{}, fmt.Errorf("abi: invalid type in call to make function type byte array.")
+	}
+	if garbage := binary.BigEndian.Uint64(word[24:32]); garbage != 0 {
+		err = fmt.Errorf("abi: got improperly encoded function type, got %v", word)
+	} else {
+		copy(funcTy[:], word[0:24])
+	}
+	return
+}
+
+// through reflection, creates a fixed array to be read from
+func readFixedBytes(t Type, word []byte) (interface{}, error) {
+	if t.T != FixedBytesTy {
+		return nil, fmt.Errorf("abi: invalid type in call to make fixed byte array.")
+	}
+	// convert
+	array := reflect.New(t.Type).Elem()
+
+	reflect.Copy(array, reflect.ValueOf(word[0:t.Size]))
+	return array.Interface(), nil
 
 }
 
-// toGoType parses the input and casts it to the proper type defined by the ABI
-// argument in T.
-func toGoType(i int, t Argument, output []byte) (interface{}, error) {
-	// we need to treat slices differently
-	if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy {
-		return toGoSlice(i, t, output)
+// iteratively unpack elements
+func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
+	if start+32*size > len(output) {
+		return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
 	}
 
-	index := i * 32
+	// this value will become our slice or our array, depending on the type
+	var refSlice reflect.Value
+	slice := output[start : start+size*32]
+
+	if t.T == SliceTy {
+		// declare our slice
+		refSlice = reflect.MakeSlice(t.Type, size, size)
+	} else if t.T == ArrayTy {
+		// declare our array
+		refSlice = reflect.New(t.Type).Elem()
+	} else {
+		return nil, fmt.Errorf("abi: invalid type in array/slice unpacking stage")
+	}
+
+	for i, j := start, 0; j*32 < len(slice); i, j = i+32, j+1 {
+		// this corrects the arrangement so that we get all the underlying array values
+		if t.Elem.T == ArrayTy && j != 0 {
+			i = start + t.Elem.Size*32*j
+		}
+		inter, err := toGoType(i, *t.Elem, output)
+		if err != nil {
+			return nil, err
+		}
+		// append the item to our reflect slice
+		refSlice.Index(j).Set(reflect.ValueOf(inter))
+	}
+
+	// return the interface
+	return refSlice.Interface(), nil
+}
+
+// toGoType parses the output bytes and recursively assigns the value of these bytes
+// into a go type with accordance with the ABI spec.
+func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	if index+32 > len(output) {
 		return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32)
 	}
 
-	// Parse the given index output and check whether we need to read
-	// a different offset and length based on the type (i.e. string, bytes)
-	var returnOutput []byte
-	switch t.Type.T {
-	case StringTy, BytesTy: // variable arrays are written at the end of the return bytes
-		// parse offset from which we should start reading
-		offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32)
-		}
-		// parse the size up until we should be reading
-		size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
-		if offset+32+size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size)
-		}
+	var (
+		returnOutput []byte
+		begin, end   int
+		err          error
+	)
 
-		// get the bytes for this return value
-		returnOutput = output[offset+32 : offset+32+size]
-	default:
+	// if we require a length prefix, find the beginning word and size returned.
+	if t.requiresLengthPrefix() {
+		begin, end, err = lengthPrefixPointsTo(index, output)
+		if err != nil {
+			return nil, err
+		}
+	} else {
 		returnOutput = output[index : index+32]
 	}
 
-	// convert the bytes to whatever is specified by the ABI.
-	switch t.Type.T {
+	switch t.T {
+	case SliceTy:
+		return forEachUnpack(t, output, begin, end)
+	case ArrayTy:
+		return forEachUnpack(t, output, index, t.Size)
+	case StringTy: // variable arrays are written at the end of the return bytes
+		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Type.Kind, returnOutput), nil
+		return readInteger(t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:
 		return common.BytesToHash(returnOutput), nil
-	case BytesTy, FixedBytesTy, FunctionTy:
-		return returnOutput, nil
-	case StringTy:
-		return string(returnOutput), nil
+	case BytesTy:
+		return output[begin : begin+end], nil
+	case FixedBytesTy:
+		return readFixedBytes(t, returnOutput)
+	case FunctionTy:
+		return readFunctionType(t, returnOutput)
+	default:
+		return nil, fmt.Errorf("abi: unknown type %v", t.T)
+	}
+}
+
+// interprets a 32 byte slice as an offset and then determines which indice to look to decode the type.
+func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) {
+	offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
+	if offset+32 > len(output) {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
+	}
+	length = int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
+	if offset+32+length > len(output) {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length)
+	}
+	start = offset + 32
+
+	//fmt.Printf("LENGTH PREFIX INFO: \nsize: %v\noffset: %v\nstart: %v\n", length, offset, start)
+	return
+}
+
+// checks for proper formatting of byte output
+func bytesAreProper(output []byte) error {
+	if len(output) == 0 {
+		return fmt.Errorf("abi: unmarshalling empty output")
+	} else if len(output)%32 != 0 {
+		return fmt.Errorf("abi: improperly formatted output")
+	} else {
+		return nil
 	}
-	return nil, fmt.Errorf("abi: unknown type %v", t.Type.T)
 }

accounts/abi/unpack_test.go

@@ -18,6 +18,7 @@ package abi
 
 import (
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"math/big"
 	"reflect"
@@ -27,260 +28,258 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-func TestSimpleMethodUnpack(t *testing.T) {
-	for i, test := range []struct {
-		def              string      // definition of the **output** ABI params
-		marshalledOutput []byte      // evm return data
-		expectedOut      interface{} // the expected output
-		outVar           string      // the output variable (e.g. uint32, *big.Int, etc)
-		err              string      // empty or error if expected
-	}{
-		{
-			`[ { "type": "bool" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			bool(true),
-			"bool",
-			"",
-		},
-		{
-			`[ { "type": "uint32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			uint32(1),
-			"uint32",
-			"",
-		},
-		{
-			`[ { "type": "uint32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"uint16",
-			"abi: cannot unmarshal uint32 in to uint16",
-		},
-		{
-			`[ { "type": "uint17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"uint16",
-			"abi: cannot unmarshal *big.Int in to uint16",
-		},
-		{
-			`[ { "type": "uint17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			big.NewInt(1),
-			"*big.Int",
-			"",
-		},
+type unpackTest struct {
+	def  string      // ABI definition JSON
+	enc  string      // evm return data
+	want interface{} // the expected output
+	err  string      // empty or error if expected
+}
 
-		{
-			`[ { "type": "int32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			int32(1),
-			"int32",
-			"",
-		},
-		{
-			`[ { "type": "int32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"int16",
-			"abi: cannot unmarshal int32 in to int16",
-		},
-		{
-			`[ { "type": "int17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"int16",
-			"abi: cannot unmarshal *big.Int in to int16",
-		},
-		{
-			`[ { "type": "int17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			big.NewInt(1),
-			"*big.Int",
-			"",
-		},
+func (test unpackTest) checkError(err error) error {
+	if err != nil {
+		if len(test.err) == 0 {
+			return fmt.Errorf("expected no err but got: %v", err)
+		} else if err.Error() != test.err {
+			return fmt.Errorf("expected err: '%v' got err: %q", test.err, err)
+		}
+	} else if len(test.err) > 0 {
+		return fmt.Errorf("expected err: %v but got none", test.err)
+	}
+	return nil
+}
 
-		{
-			`[ { "type": "address" } ]`,
-			common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000"),
-			common.Address{1},
-			"address",
-			"",
-		},
-		{
-			`[ { "type": "bytes32" } ]`,
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			"bytes",
-			"",
-		},
-		{
-			`[ { "type": "bytes32" } ]`,
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			"hash",
-			"",
-		},
-		{
-			`[ { "type": "bytes32" } ]`,
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			"interface",
-			"",
-		},
-		{
-			`[ { "type": "function" } ]`,
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-			[24]byte{1},
-			"function",
-			"",
-		},
-	} {
-		abiDefinition := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
-		abi, err := JSON(strings.NewReader(abiDefinition))
+var unpackTests = []unpackTest{
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: true,
+	},
+	{
+		def:  `[{"type": "uint32"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: uint32(1),
+	},
+	{
+		def:  `[{"type": "uint32"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: uint16(0),
+		err:  "abi: cannot unmarshal uint32 in to uint16",
+	},
+	{
+		def:  `[{"type": "uint17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: uint16(0),
+		err:  "abi: cannot unmarshal *big.Int in to uint16",
+	},
+	{
+		def:  `[{"type": "uint17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: big.NewInt(1),
+	},
+	{
+		def:  `[{"type": "int32"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: int32(1),
+	},
+	{
+		def:  `[{"type": "int32"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: int16(0),
+		err:  "abi: cannot unmarshal int32 in to int16",
+	},
+	{
+		def:  `[{"type": "int17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: int16(0),
+		err:  "abi: cannot unmarshal *big.Int in to int16",
+	},
+	{
+		def:  `[{"type": "int17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: big.NewInt(1),
+	},
+	{
+		def:  `[{"type": "address"}]`,
+		enc:  "0000000000000000000000000100000000000000000000000000000000000000",
+		want: common.Address{1},
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: [32]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+	},
+	{
+		def:  `[{"type": "bytes"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
+	{
+		def:  `[{"type": "bytes"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: [32]byte{},
+		err:  "abi: cannot unmarshal []uint8 in to [32]uint8",
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: []byte(nil),
+		err:  "abi: cannot unmarshal [32]uint8 in to []uint8",
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: common.HexToHash("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
+	{
+		def:  `[{"type": "function"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: [24]byte{1},
+	},
+	// slices
+	{
+		def:  `[{"type": "uint8[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint8{1, 2},
+	},
+	{
+		def:  `[{"type": "uint8[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint8{1, 2},
+	},
+	// multi dimensional, if these pass, all types that don't require length prefix should pass
+	{
+		def:  `[{"type": "uint8[][]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000E0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [][]uint8{{1, 2}, {1, 2}},
+	},
+	{
+		def:  `[{"type": "uint8[2][2]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2][2]uint8{{1, 2}, {1, 2}},
+	},
+	{
+		def:  `[{"type": "uint8[][2]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001",
+		want: [2][]uint8{{1}, {1}},
+	},
+	{
+		def:  `[{"type": "uint8[2][]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [][2]uint8{{1, 2}},
+	},
+	{
+		def:  `[{"type": "uint16[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint16{1, 2},
+	},
+	{
+		def:  `[{"type": "uint16[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint16{1, 2},
+	},
+	{
+		def:  `[{"type": "uint32[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint32{1, 2},
+	},
+	{
+		def:  `[{"type": "uint32[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint32{1, 2},
+	},
+	{
+		def:  `[{"type": "uint64[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint64{1, 2},
+	},
+	{
+		def:  `[{"type": "uint64[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint64{1, 2},
+	},
+	{
+		def:  `[{"type": "uint256[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []*big.Int{big.NewInt(1), big.NewInt(2)},
+	},
+	{
+		def:  `[{"type": "uint256[3]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		want: [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
+	},
+	{
+		def:  `[{"type": "int8[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int8{1, 2},
+	},
+	{
+		def:  `[{"type": "int8[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int8{1, 2},
+	},
+	{
+		def:  `[{"type": "int16[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int16{1, 2},
+	},
+	{
+		def:  `[{"type": "int16[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int16{1, 2},
+	},
+	{
+		def:  `[{"type": "int32[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int32{1, 2},
+	},
+	{
+		def:  `[{"type": "int32[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int32{1, 2},
+	},
+	{
+		def:  `[{"type": "int64[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int64{1, 2},
+	},
+	{
+		def:  `[{"type": "int64[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int64{1, 2},
+	},
+	{
+		def:  `[{"type": "int256[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []*big.Int{big.NewInt(1), big.NewInt(2)},
+	},
+	{
+		def:  `[{"type": "int256[3]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		want: [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
+	},
+}
+
+func TestUnpack(t *testing.T) {
+	for i, test := range unpackTests {
+		def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
+		abi, err := JSON(strings.NewReader(def))
 		if err != nil {
-			t.Errorf("%d failed. %v", i, err)
+			t.Fatalf("invalid ABI definition %s: %v", def, err)
+		}
+		encb, err := hex.DecodeString(test.enc)
+		if err != nil {
+			t.Fatalf("invalid hex: %s" + test.enc)
+		}
+		outptr := reflect.New(reflect.TypeOf(test.want))
+		err = abi.Unpack(outptr.Interface(), "method", encb)
+		if err := test.checkError(err); err != nil {
+			t.Errorf("test %d (%v) failed: %v", i, test.def, err)
 			continue
 		}
-
-		var outvar interface{}
-		switch test.outVar {
-		case "bool":
-			var v bool
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint8":
-			var v uint8
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint16":
-			var v uint16
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint32":
-			var v uint32
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint64":
-			var v uint64
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int8":
-			var v int8
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int16":
-			var v int16
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int32":
-			var v int32
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int64":
-			var v int64
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "*big.Int":
-			var v *big.Int
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "address":
-			var v common.Address
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "bytes":
-			var v []byte
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "hash":
-			var v common.Hash
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v.Bytes()[:]
-		case "function":
-			var v [24]byte
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "interface":
-			err = abi.Unpack(&outvar, "method", test.marshalledOutput)
-		default:
-			t.Errorf("unsupported type '%v' please add it to the switch statement in this test", test.outVar)
-			continue
+		out := outptr.Elem().Interface()
+		if !reflect.DeepEqual(test.want, out) {
+			t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out)
 		}
-
-		if err != nil && len(test.err) == 0 {
-			t.Errorf("%d failed. Expected no err but got: %v", i, err)
-			continue
-		}
-		if err == nil && len(test.err) != 0 {
-			t.Errorf("%d failed. Expected err: %v but got none", i, test.err)
-			continue
-		}
-		if err != nil && len(test.err) != 0 && err.Error() != test.err {
-			t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err)
-			continue
-		}
-
-		if err == nil {
-			if !reflect.DeepEqual(test.expectedOut, outvar) {
-				t.Errorf("%d failed. Output error: expected %v, got %v", i, test.expectedOut, outvar)
-			}
-		}
-	}
-}
-
-func TestUnpackSetInterfaceSlice(t *testing.T) {
-	var (
-		var1 = new(uint8)
-		var2 = new(uint8)
-	)
-	out := []interface{}{var1, var2}
-	abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint8"}, {"type":"uint8"}]}]`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...)
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if *var1 != 1 {
-		t.Error("expected var1 to be 1, got", *var1)
-	}
-	if *var2 != 2 {
-		t.Error("expected var2 to be 2, got", *var2)
-	}
-
-	out = []interface{}{var1}
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-
-	expErr := "abi: cannot marshal in to slices of unequal size (require: 2, got: 1)"
-	if err == nil || err.Error() != expErr {
-		t.Error("expected err:", expErr, "Got:", err)
-	}
-}
-
-func TestUnpackSetInterfaceArrayOutput(t *testing.T) {
-	var (
-		var1 = new([1]uint32)
-		var2 = new([1]uint32)
-	)
-	out := []interface{}{var1, var2}
-	abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint32[1]"}, {"type":"uint32[1]"}]}]`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...)
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if *var1 != [1]uint32{1} {
-		t.Error("expected var1 to be [1], got", *var1)
-	}
-	if *var2 != [1]uint32{2} {
-		t.Error("expected var2 to be [2], got", *var2)
 	}
 }
 
@@ -337,101 +336,6 @@ func TestMultiReturnWithStruct(t *testing.T) {
 	}
 }
 
-func TestMultiReturnWithSlice(t *testing.T) {
-	const definition = `[
-	{ "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
-
-	abi, err := JSON(strings.NewReader(definition))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// using buff to make the code readable
-	buff := new(bytes.Buffer)
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005"))
-	stringOut := "hello"
-	buff.Write(common.RightPadBytes([]byte(stringOut), 32))
-
-	var inter []interface{}
-	err = abi.Unpack(&inter, "multi", buff.Bytes())
-	if err != nil {
-		t.Error(err)
-	}
-
-	if len(inter) != 2 {
-		t.Fatal("expected 2 results got", len(inter))
-	}
-
-	if num, ok := inter[0].(*big.Int); !ok || num.Cmp(big.NewInt(1)) != 0 {
-		t.Error("expected index 0 to be 1 got", num)
-	}
-
-	if str, ok := inter[1].(string); !ok || str != stringOut {
-		t.Error("expected index 1 to be", stringOut, "got", str)
-	}
-}
-
-func TestMarshalArrays(t *testing.T) {
-	const definition = `[
-	{ "name" : "bytes32", "constant" : false, "outputs": [ { "type": "bytes32" } ] },
-	{ "name" : "bytes10", "constant" : false, "outputs": [ { "type": "bytes10" } ] }
-	]`
-
-	abi, err := JSON(strings.NewReader(definition))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	output := common.LeftPadBytes([]byte{1}, 32)
-
-	var bytes10 [10]byte
-	err = abi.Unpack(&bytes10, "bytes32", output)
-	if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" {
-		t.Error("expected error or bytes32 not be assignable to bytes10:", err)
-	}
-
-	var bytes32 [32]byte
-	err = abi.Unpack(&bytes32, "bytes32", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(bytes32[:], output) {
-		t.Error("expected bytes32[31] to be 1 got", bytes32[31])
-	}
-
-	type (
-		B10 [10]byte
-		B32 [32]byte
-	)
-
-	var b10 B10
-	err = abi.Unpack(&b10, "bytes32", output)
-	if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" {
-		t.Error("expected error or bytes32 not be assignable to bytes10:", err)
-	}
-
-	var b32 B32
-	err = abi.Unpack(&b32, "bytes32", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(b32[:], output) {
-		t.Error("expected bytes32[31] to be 1 got", bytes32[31])
-	}
-
-	output[10] = 1
-	var shortAssignLong [32]byte
-	err = abi.Unpack(&shortAssignLong, "bytes10", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(output, shortAssignLong[:]) {
-		t.Errorf("expected %x to be %x", shortAssignLong, output)
-	}
-}
-
 func TestUnmarshal(t *testing.T) {
 	const definition = `[
 	{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },
@@ -450,6 +354,29 @@ func TestUnmarshal(t *testing.T) {
 	}
 	buff := new(bytes.Buffer)
 
+	// marshall mixed bytes (mixedBytes)
+	p0, p0Exp := []byte{}, common.Hex2Bytes("01020000000000000000")
+	p1, p1Exp := [32]byte{}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff")
+	mixedBytes := []interface{}{&p0, &p1}
+
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff"))
+	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000a"))
+	buff.Write(common.Hex2Bytes("0102000000000000000000000000000000000000000000000000000000000000"))
+
+	err = abi.Unpack(&mixedBytes, "mixedBytes", buff.Bytes())
+	if err != nil {
+		t.Error(err)
+	} else {
+		if bytes.Compare(p0, p0Exp) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", p0Exp, p0)
+		}
+
+		if bytes.Compare(p1[:], p1Exp) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", p1Exp, p1)
+		}
+	}
+
 	// marshal int
 	var Int *big.Int
 	err = abi.Unpack(&Int, "int", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
@@ -473,6 +400,7 @@ func TestUnmarshal(t *testing.T) {
 	}
 
 	// marshal dynamic bytes max length 32
+	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	bytesOut := common.RightPadBytes([]byte("hello"), 32)
@@ -504,11 +432,11 @@ func TestUnmarshal(t *testing.T) {
 		t.Errorf("expected %x got %x", bytesOut, Bytes)
 	}
 
-	// marshall dynamic bytes max length 63
+	// marshall dynamic bytes max length 64
 	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f"))
-	bytesOut = common.RightPadBytes([]byte("hello"), 63)
+	bytesOut = common.RightPadBytes([]byte("hello"), 64)
 	buff.Write(bytesOut)
 
 	err = abi.Unpack(&Bytes, "bytes", buff.Bytes())
@@ -516,8 +444,8 @@ func TestUnmarshal(t *testing.T) {
 		t.Error(err)
 	}
 
-	if !bytes.Equal(Bytes, bytesOut) {
-		t.Errorf("expected %x got %x", bytesOut, Bytes)
+	if !bytes.Equal(Bytes, bytesOut[:len(bytesOut)-1]) {
+		t.Errorf("expected %x got %x", bytesOut[:len(bytesOut)-1], Bytes)
 	}
 
 	// marshal dynamic bytes output empty
@@ -569,29 +497,6 @@ func TestUnmarshal(t *testing.T) {
 		t.Error("expected error")
 	}
 
-	// marshal mixed bytes
-	buff.Reset()
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
-	fixed := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")
-	buff.Write(fixed)
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
-	bytesOut = common.RightPadBytes([]byte("hello"), 32)
-	buff.Write(bytesOut)
-
-	var out []interface{}
-	err = abi.Unpack(&out, "mixedBytes", buff.Bytes())
-	if err != nil {
-		t.Fatal("didn't expect error:", err)
-	}
-
-	if !bytes.Equal(bytesOut, out[0].([]byte)) {
-		t.Errorf("expected %x, got %x", bytesOut, out[0])
-	}
-
-	if !bytes.Equal(fixed, out[1].([]byte)) {
-		t.Errorf("expected %x, got %x", fixed, out[1])
-	}
-
 	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"))

accounts/keystore/account_cache.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -31,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/fatih/set.v0"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -71,6 +71,7 @@ type accountCache struct {
 	byAddr   map[common.Address][]accounts.Account
 	throttle *time.Timer
 	notify   chan struct{}
+	fileC    fileCache
 }
 
 func newAccountCache(keydir string) (*accountCache, chan struct{}) {
@@ -78,6 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
 		keydir: keydir,
 		byAddr: make(map[common.Address][]accounts.Account),
 		notify: make(chan struct{}, 1),
+		fileC:  fileCache{all: set.NewNonTS()},
 	}
 	ac.watcher = newWatcher(ac)
 	return ac, ac.notify
@@ -127,6 +129,23 @@ func (ac *accountCache) delete(removed accounts.Account) {
 	}
 }
 
+// deleteByFile removes an account referenced by the given path.
+func (ac *accountCache) deleteByFile(path string) {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Path >= path })
+
+	if i < len(ac.all) && ac.all[i].URL.Path == path {
+		removed := ac.all[i]
+		ac.all = append(ac.all[:i], ac.all[i+1:]...)
+		if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
+			delete(ac.byAddr, removed.Address)
+		} else {
+			ac.byAddr[removed.Address] = ba
+		}
+	}
+}
+
 func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
@@ -167,15 +186,16 @@ func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
 	default:
 		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
 		copy(err.Matches, matches)
+		sort.Sort(accountsByURL(err.Matches))
 		return accounts.Account{}, err
 	}
 }
 
 func (ac *accountCache) maybeReload() {
 	ac.mu.Lock()
-	defer ac.mu.Unlock()
 
 	if ac.watcher.running {
+		ac.mu.Unlock()
 		return // A watcher is running and will keep the cache up-to-date.
 	}
 	if ac.throttle == nil {
@@ -184,12 +204,15 @@ func (ac *accountCache) maybeReload() {
 		select {
 		case <-ac.throttle.C:
 		default:
+			ac.mu.Unlock()
 			return // The cache was reloaded recently.
 		}
 	}
+	// No watcher running, start it.
 	ac.watcher.start()
-	ac.reload()
 	ac.throttle.Reset(minReloadInterval)
+	ac.mu.Unlock()
+	ac.scanAccounts()
 }
 
 func (ac *accountCache) close() {
@@ -205,80 +228,71 @@ func (ac *accountCache) close() {
 	ac.mu.Unlock()
 }
 
-// reload caches addresses of existing accounts.
-// Callers must hold ac.mu.
-func (ac *accountCache) reload() {
-	accounts, err := ac.scan()
+// scanAccounts checks if any changes have occurred on the filesystem, and
+// updates the account cache accordingly
+func (ac *accountCache) scanAccounts() error {
+	// Scan the entire folder metadata for file changes
+	creates, deletes, updates, err := ac.fileC.scan(ac.keydir)
 	if err != nil {
 		log.Debug("Failed to reload keystore contents", "err", err)
+		return err
 	}
-	ac.all = accounts
-	sort.Sort(ac.all)
-	for k := range ac.byAddr {
-		delete(ac.byAddr, k)
+	if creates.Size() == 0 && deletes.Size() == 0 && updates.Size() == 0 {
+		return nil
 	}
-	for _, a := range accounts {
-		ac.byAddr[a.Address] = append(ac.byAddr[a.Address], a)
+	// Create a helper method to scan the contents of the key files
+	var (
+		buf = new(bufio.Reader)
+		key struct {
+			Address string `json:"address"`
+		}
+	)
+	readAccount := func(path string) *accounts.Account {
+		fd, err := os.Open(path)
+		if err != nil {
+			log.Trace("Failed to open keystore file", "path", path, "err", err)
+			return nil
+		}
+		defer fd.Close()
+		buf.Reset(fd)
+		// Parse the address.
+		key.Address = ""
+		err = json.NewDecoder(buf).Decode(&key)
+		addr := common.HexToAddress(key.Address)
+		switch {
+		case err != nil:
+			log.Debug("Failed to decode keystore key", "path", path, "err", err)
+		case (addr == common.Address{}):
+			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
+		default:
+			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+		}
+		return nil
 	}
+	// Process all the file diffs
+	start := time.Now()
+
+	for _, p := range creates.List() {
+		if a := readAccount(p.(string)); a != nil {
+			ac.add(*a)
+		}
+	}
+	for _, p := range deletes.List() {
+		ac.deleteByFile(p.(string))
+	}
+	for _, p := range updates.List() {
+		path := p.(string)
+		ac.deleteByFile(path)
+		if a := readAccount(path); a != nil {
+			ac.add(*a)
+		}
+	}
+	end := time.Now()
+
 	select {
 	case ac.notify <- struct{}{}:
 	default:
 	}
-	log.Debug("Reloaded keystore contents", "accounts", len(ac.all))
-}
-
-func (ac *accountCache) scan() ([]accounts.Account, error) {
-	files, err := ioutil.ReadDir(ac.keydir)
-	if err != nil {
-		return nil, err
-	}
-
-	var (
-		buf     = new(bufio.Reader)
-		addrs   []accounts.Account
-		keyJSON struct {
-			Address string `json:"address"`
-		}
-	)
-	for _, fi := range files {
-		path := filepath.Join(ac.keydir, fi.Name())
-		if skipKeyFile(fi) {
-			log.Trace("Ignoring file on account scan", "path", path)
-			continue
-		}
-		logger := log.New("path", path)
-
-		fd, err := os.Open(path)
-		if err != nil {
-			logger.Trace("Failed to open keystore file", "err", err)
-			continue
-		}
-		buf.Reset(fd)
-		// Parse the address.
-		keyJSON.Address = ""
-		err = json.NewDecoder(buf).Decode(&keyJSON)
-		addr := common.HexToAddress(keyJSON.Address)
-		switch {
-		case err != nil:
-			logger.Debug("Failed to decode keystore key", "err", err)
-		case (addr == common.Address{}):
-			logger.Debug("Failed to decode keystore key", "err", "missing or zero address")
-		default:
-			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
-		}
-		fd.Close()
-	}
-	return addrs, err
-}
-
-func skipKeyFile(fi os.FileInfo) bool {
-	// Skip editor backups and UNIX-style hidden files.
-	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
-		return true
-	}
-	// Skip misc special files, directories (yes, symlinks too).
-	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
-		return true
-	}
-	return false
+	log.Trace("Handled keystore changes", "time", end.Sub(start))
+	return nil
 }

accounts/keystore/account_cache_test.go

@@ -18,6 +18,7 @@ package keystore
 
 import (
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -295,3 +296,101 @@ func TestCacheFind(t *testing.T) {
 		}
 	}
 }
+
+func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
+	var list []accounts.Account
+	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
+		list = ks.Accounts()
+		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				return fmt.Errorf("wasn't notified of new accounts")
+			}
+			return nil
+		}
+		time.Sleep(d)
+	}
+	return fmt.Errorf("\ngot  %v\nwant %v", list, wantAccounts)
+}
+
+// TestUpdatedKeyfileContents tests that updating the contents of a keystore file
+// is noticed by the watcher, and the account cache is updated accordingly
+func TestUpdatedKeyfileContents(t *testing.T) {
+	t.Parallel()
+
+	// Create a temporary kesytore to test with
+	rand.Seed(time.Now().UnixNano())
+	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int()))
+	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
+
+	list := ks.Accounts()
+	if len(list) > 0 {
+		t.Error("initial account list not empty:", list)
+	}
+	time.Sleep(100 * time.Millisecond)
+
+	// Create the directory and copy a key file into it.
+	os.MkdirAll(dir, 0700)
+	defer os.RemoveAll(dir)
+	file := filepath.Join(dir, "aaa")
+
+	// Place one of our testfiles in there
+	if err := cp.CopyFile(file, cachetestAccounts[0].URL.Path); err != nil {
+		t.Fatal(err)
+	}
+
+	// ks should see the account.
+	wantAccounts := []accounts.Account{cachetestAccounts[0]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Now replace file contents
+	if err := forceCopyFile(file, cachetestAccounts[1].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[1]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("First replacement failed")
+		t.Error(err)
+		return
+	}
+
+	// Now replace file contents again
+	if err := forceCopyFile(file, cachetestAccounts[2].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[2]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("Second replacement failed")
+		t.Error(err)
+		return
+	}
+	// Now replace file contents with crap
+	if err := ioutil.WriteFile(file, []byte("foo"), 0644); err != nil {
+		t.Fatal(err)
+		return
+	}
+	if err := waitForAccounts([]accounts.Account{}, ks); err != nil {
+		t.Errorf("Emptying account file failed")
+		t.Error(err)
+		return
+	}
+}
+
+// forceCopyFile is like cp.CopyFile, but doesn't complain if the destination exists.
+func forceCopyFile(dst, src string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(dst, data, 0644)
+}

accounts/keystore/file_cache.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	set "gopkg.in/fatih/set.v0"
+)
+
+// fileCache is a cache of files seen during scan of keystore.
+type fileCache struct {
+	all     *set.SetNonTS // Set of all files from the keystore folder
+	lastMod time.Time     // Last time instance when a file was modified
+	mu      sync.RWMutex
+}
+
+// scan performs a new scan on the given directory, compares against the already
+// cached filenames, and returns file sets: creates, deletes, updates.
+func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Interface, error) {
+	t0 := time.Now()
+
+	// List all the failes from the keystore folder
+	files, err := ioutil.ReadDir(keyDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	t1 := time.Now()
+
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	// Iterate all the files and gather their metadata
+	all := set.NewNonTS()
+	mods := set.NewNonTS()
+
+	var newLastMod time.Time
+	for _, fi := range files {
+		// Skip any non-key files from the folder
+		path := filepath.Join(keyDir, fi.Name())
+		if skipKeyFile(fi) {
+			log.Trace("Ignoring file on account scan", "path", path)
+			continue
+		}
+		// Gather the set of all and fresly modified files
+		all.Add(path)
+
+		modified := fi.ModTime()
+		if modified.After(fc.lastMod) {
+			mods.Add(path)
+		}
+		if modified.After(newLastMod) {
+			newLastMod = modified
+		}
+	}
+	t2 := time.Now()
+
+	// Update the tracked files and return the three sets
+	deletes := set.Difference(fc.all, all)   // Deletes = previous - current
+	creates := set.Difference(all, fc.all)   // Creates = current - previous
+	updates := set.Difference(mods, creates) // Updates = modified - creates
+
+	fc.all, fc.lastMod = all, newLastMod
+	t3 := time.Now()
+
+	// Report on the scanning stats and return
+	log.Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2))
+	return creates, deletes, updates, nil
+}
+
+// skipKeyFile ignores editor backups, hidden files and folders/symlinks.
+func skipKeyFile(fi os.FileInfo) bool {
+	// Skip editor backups and UNIX-style hidden files.
+	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
+		return true
+	}
+	// Skip misc special files, directories (yes, symlinks too).
+	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
+		return true
+	}
+	return false
+}

accounts/keystore/keystore_passphrase.go

@@ -28,6 +28,7 @@ package keystore
 import (
 	"bytes"
 	"crypto/aes"
+	crand "crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -90,6 +91,12 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 	return key, nil
 }
 
+// StoreKey generates a key, encrypts with 'auth' and stores in the given directory
+func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, crand.Reader, auth)
+	return a.Address, err
+}
+
 func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) error {
 	keyjson, err := EncryptKey(key, auth, ks.scryptN, ks.scryptP)
 	if err != nil {

accounts/keystore/keystore_test.go

@@ -272,82 +272,104 @@ func TestWalletNotifierLifecycle(t *testing.T) {
 	t.Errorf("wallet notifier didn't terminate after unsubscribe")
 }
 
+type walletEvent struct {
+	accounts.WalletEvent
+	a accounts.Account
+}
+
 // Tests that wallet notifications and correctly fired when accounts are added
 // or deleted from the keystore.
 func TestWalletNotifications(t *testing.T) {
-	// Create a temporary kesytore to test with
 	dir, ks := tmpKeyStore(t, false)
 	defer os.RemoveAll(dir)
 
-	// Subscribe to the wallet feed
-	updates := make(chan accounts.WalletEvent, 1)
-	sub := ks.Subscribe(updates)
+	// Subscribe to the wallet feed and collect events.
+	var (
+		events  []walletEvent
+		updates = make(chan accounts.WalletEvent)
+		sub     = ks.Subscribe(updates)
+	)
 	defer sub.Unsubscribe()
+	go func() {
+		for {
+			select {
+			case ev := <-updates:
+				events = append(events, walletEvent{ev, ev.Wallet.Accounts()[0]})
+			case <-sub.Err():
+				close(updates)
+				return
+			}
+		}
+	}()
 
-	// Randomly add and remove account and make sure events and wallets are in sync
-	live := make(map[common.Address]accounts.Account)
+	// Randomly add and remove accounts.
+	var (
+		live       = make(map[common.Address]accounts.Account)
+		wantEvents []walletEvent
+	)
 	for i := 0; i < 1024; i++ {
-		// Execute a creation or deletion and ensure event arrival
 		if create := len(live) == 0 || rand.Int()%4 > 0; create {
 			// Add a new account and ensure wallet notifications arrives
 			account, err := ks.NewAccount("")
 			if err != nil {
 				t.Fatalf("failed to create test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if event.Kind != accounts.WalletArrived {
-					t.Errorf("non-arrival event on account creation")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on created wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet arrival event not fired on account creation")
-			}
 			live[account.Address] = account
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletArrived}, account})
 		} else {
-			// Select a random account to delete (crude, but works)
+			// Delete a random account.
 			var account accounts.Account
 			for _, a := range live {
 				account = a
 				break
 			}
-			// Remove an account and ensure wallet notifiaction arrives
 			if err := ks.Delete(account, ""); err != nil {
 				t.Fatalf("failed to delete test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if event.Kind != accounts.WalletDropped {
-					t.Errorf("non-drop event on account deletion")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on deleted wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet departure event not fired on account creation")
-			}
 			delete(live, account.Address)
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletDropped}, account})
 		}
-		// Retrieve the list of wallets and ensure it matches with our required live set
-		liveList := make([]accounts.Account, 0, len(live))
-		for _, account := range live {
-			liveList = append(liveList, account)
-		}
-		sort.Sort(accountsByURL(liveList))
+	}
 
-		wallets := ks.Wallets()
-		if len(liveList) != len(wallets) {
-			t.Errorf("wallet list doesn't match required accounts: have %v, want %v", wallets, liveList)
-		} else {
-			for j, wallet := range wallets {
-				if accs := wallet.Accounts(); len(accs) != 1 {
-					t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
-				} else if accs[0] != liveList[j] {
-					t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
-				}
+	// Shut down the event collector and check events.
+	sub.Unsubscribe()
+	<-updates
+	checkAccounts(t, live, ks.Wallets())
+	checkEvents(t, wantEvents, events)
+}
+
+// checkAccounts checks that all known live accounts are present in the wallet list.
+func checkAccounts(t *testing.T, live map[common.Address]accounts.Account, wallets []accounts.Wallet) {
+	if len(live) != len(wallets) {
+		t.Errorf("wallet list doesn't match required accounts: have %d, want %d", len(wallets), len(live))
+		return
+	}
+	liveList := make([]accounts.Account, 0, len(live))
+	for _, account := range live {
+		liveList = append(liveList, account)
+	}
+	sort.Sort(accountsByURL(liveList))
+	for j, wallet := range wallets {
+		if accs := wallet.Accounts(); len(accs) != 1 {
+			t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+		} else if accs[0] != liveList[j] {
+			t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+		}
+	}
+}
+
+// checkEvents checks that all events in 'want' are present in 'have'. Events may be present multiple times.
+func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
+	for _, wantEv := range want {
+		nmatch := 0
+		for ; len(have) > 0; nmatch++ {
+			if have[0].Kind != wantEv.Kind || have[0].a != wantEv.a {
+				break
 			}
+			have = have[1:]
+		}
+		if nmatch == 0 {
+			t.Fatalf("can't find event with Kind=%v for %x", wantEv.Kind, wantEv.a.Address)
 		}
 	}
 }

accounts/keystore/watch.go

@@ -70,7 +70,6 @@ func (w *watcher) loop() {
 		return
 	}
 	defer notify.Stop(w.ev)
-
 	logger.Trace("Started watching keystore folder")
 	defer logger.Trace("Stopped watching keystore folder")
 
@@ -82,32 +81,28 @@ func (w *watcher) loop() {
 	// When an event occurs, the reload call is delayed a bit so that
 	// multiple events arriving quickly only cause a single reload.
 	var (
-		debounce          = time.NewTimer(0)
-		debounceDuration  = 500 * time.Millisecond
-		inCycle, hadEvent bool
+		debounceDuration = 500 * time.Millisecond
+		rescanTriggered  = false
+		debounce         = time.NewTimer(0)
 	)
+	// Ignore initial trigger
+	if !debounce.Stop() {
+		<-debounce.C
+	}
 	defer debounce.Stop()
 	for {
 		select {
 		case <-w.quit:
 			return
 		case <-w.ev:
-			if !inCycle {
+			// Trigger the scan (with delay), if not already triggered
+			if !rescanTriggered {
 				debounce.Reset(debounceDuration)
-				inCycle = true
-			} else {
-				hadEvent = true
+				rescanTriggered = true
 			}
 		case <-debounce.C:
-			w.ac.mu.Lock()
-			w.ac.reload()
-			w.ac.mu.Unlock()
-			if hadEvent {
-				debounce.Reset(debounceDuration)
-				inCycle, hadEvent = true, false
-			} else {
-				inCycle, hadEvent = false, false
-			}
+			w.ac.scanAccounts()
+			rescanTriggered = false
 		}
 	}
 }

accounts/manager.go

@@ -41,6 +41,11 @@ type Manager struct {
 // NewManager creates a generic account manager to sign transaction via various
 // supported backends.
 func NewManager(backends ...Backend) *Manager {
+	// Retrieve the initial list of wallets from the backends and sort by URL
+	var wallets []Wallet
+	for _, backend := range backends {
+		wallets = merge(wallets, backend.Wallets()...)
+	}
 	// Subscribe to wallet notifications from all backends
 	updates := make(chan WalletEvent, 4*len(backends))
 
@@ -48,11 +53,6 @@ func NewManager(backends ...Backend) *Manager {
 	for i, backend := range backends {
 		subs[i] = backend.Subscribe(updates)
 	}
-	// Retrieve the initial list of wallets from the backends and sort by URL
-	var wallets []Wallet
-	for _, backend := range backends {
-		wallets = merge(wallets, backend.Wallets()...)
-	}
 	// Assemble the account manager and return
 	am := &Manager{
 		backends: make(map[reflect.Type][]Backend),

build/ci.go

@@ -24,7 +24,8 @@ Usage: go run ci.go <command> <command flags/arguments>
 Available commands are:
 
    install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -misspell ] [ packages... ]                                      -- runs the tests
+   test       [ -coverage ] [ packages... ]                                                    -- runs the tests
+   lint                                                                                        -- runs certain pre-selected linters
    archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
@@ -120,7 +121,7 @@ var (
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "zesty"}
+	debDistros = []string{"trusty", "xenial", "zesty", "artful"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -146,6 +147,8 @@ func main() {
 		doInstall(os.Args[2:])
 	case "test":
 		doTest(os.Args[2:])
+	case "lint":
+		doLint(os.Args[2:])
 	case "archive":
 		doArchive(os.Args[2:])
 	case "debsrc":
@@ -280,7 +283,6 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 
 func doTest(cmdline []string) {
 	var (
-		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
@@ -294,10 +296,7 @@ func doTest(cmdline []string) {
 
 	// Run analysis tools before the tests.
 	build.MustRun(goTool("vet", packages...))
-	if *misspell {
-		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
-		// spellcheck(packages)
-	}
+
 	// Run the actual tests.
 	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
@@ -306,35 +305,31 @@ func doTest(cmdline []string) {
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
+
 	gotest.Args = append(gotest.Args, packages...)
 	build.MustRun(gotest)
 }
 
-// spellcheck runs the client9/misspell spellchecker package on all Go, Cgo and
-// test files in the requested packages.
-func spellcheck(packages []string) {
-	// Ensure the spellchecker is available
-	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
+// runs gometalinter on requested packages
+func doLint(cmdline []string) {
+	flag.CommandLine.Parse(cmdline)
 
-	// Windows chokes on long argument lists, check packages individually
-	for _, pkg := range packages {
-		// The spell checker doesn't work on packages, gather all .go files for it
-		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
-		if err != nil {
-			log.Fatalf("source file listing failed: %v\n%s", err, string(out))
-		}
-		// Retrieve the folder and assemble the source list
-		lines := strings.Split(string(out), "\n")
-		root := lines[0]
+	packages := []string{"./..."}
+	if len(flag.CommandLine.Args()) > 0 {
+		packages = flag.CommandLine.Args()
+	}
+	// Get metalinter and install all supported linters
+	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v1"))
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), "--install")
 
-		sources := make([]string, 0, len(lines)-1)
-		for _, line := range lines[1:] {
-			if line = strings.TrimSpace(line); line != "" {
-				sources = append(sources, filepath.Join(root, line))
-			}
-		}
-		// Run the spell checker for this particular package
-		build.MustRunCommand(filepath.Join(GOBIN, "misspell"), append([]string{"-error"}, sources...)...)
+	// Run fast linters batched together
+	configs := []string{"--vendor", "--disable-all", "--enable=vet", "--enable=gofmt", "--enable=misspell"}
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
+
+	// Run slow linters one by one
+	for _, linter := range []string{"unconvert"} {
+		configs = []string{"--vendor", "--deadline=10m", "--disable-all", "--enable=" + linter}
+		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
 	}
 }
 

cmd/evm/staterunner.go

@@ -94,7 +94,8 @@ func stateTestCmd(ctx *cli.Context) error {
 		for _, st := range test.Subtests() {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
-			if state, err := test.Run(st, cfg); err != nil {
+			state, err := test.Run(st, cfg)
+			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
@@ -102,6 +103,11 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
+			// print state root for evmlab tracing (already committed above, so no need to delete objects again
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
+
 			results = append(results, *result)
 
 			// Print any structured logs collected

cmd/faucet/faucet.go

@@ -21,8 +21,10 @@ package main
 
 import (
 	"bytes"
+	"compress/zlib"
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"html/template"
@@ -33,6 +35,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -181,10 +184,10 @@ func main() {
 
 // request represents an accepted funding request.
 type request struct {
-	Username string             `json:"username"` // GitHub user for displaying an avatar
-	Account  common.Address     `json:"account"`  // Ethereum address being funded
-	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
-	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+	Avatar  string             `json:"avatar"`  // Avatar URL to make the UI nicer
+	Account common.Address     `json:"account"` // Ethereum address being funded
+	Time    time.Time          `json:"time"`    // Timestamp when the request was accepted
+	Tx      *types.Transaction `json:"tx"`      // Transaction funding the account
 }
 
 // faucet represents a crypto faucet backed by an Ethereum light client.
@@ -299,6 +302,8 @@ func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
 // apiHandler handles requests for Ether grants and transaction statuses.
 func (f *faucet) apiHandler(conn *websocket.Conn) {
 	// Start tracking the connection and drop at the end
+	defer conn.Close()
+
 	f.lock.Lock()
 	f.conns = append(f.conns, conn)
 	f.lock.Unlock()
@@ -313,25 +318,50 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		}
 		f.lock.Unlock()
 	}()
-	// Send a few initial stats to the client
-	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
-	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+	// Gather the initial stats from the network to report
+	var (
+		head    *types.Header
+		balance *big.Int
+		nonce   uint64
+		err     error
+	)
+	for {
+		// Attempt to retrieve the stats, may error on no faucet connectivity
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		head, err = f.client.HeaderByNumber(ctx, nil)
+		if err == nil {
+			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
+			if err == nil {
+				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
+			}
+		}
+		cancel()
 
-	websocket.JSON.Send(conn, map[string]interface{}{
+		// If stats retrieval failed, wait a bit and retry
+		if err != nil {
+			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+				log.Warn("Failed to send faucet error to client", "err", err)
+				return
+			}
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		// Initial stats reported successfully, proceed with user interaction
+		break
+	}
+	// Send over the initial stats and the latest header
+	if err = send(conn, map[string]interface{}{
 		"funds":    balance.Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
-	})
-	// Send the initial block to the client
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	header, err := f.client.HeaderByNumber(ctx, nil)
-	cancel()
-
-	if err != nil {
-		log.Error("Failed to retrieve latest header", "err", err)
-	} else {
-		websocket.JSON.Send(conn, header)
+	}, 3*time.Second); err != nil {
+		log.Warn("Failed to send initial stats to client", "err", err)
+		return
+	}
+	if err = send(conn, head, 3*time.Second); err != nil {
+		log.Warn("Failed to send initial header to client", "err", err)
+		return
 	}
 	// Keep reading requests from the websocket until the connection breaks
 	for {
@@ -341,18 +371,25 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			Tier    uint   `json:"tier"`
 			Captcha string `json:"captcha"`
 		}
-		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+		if err = websocket.JSON.Receive(conn, &msg); err != nil {
 			return
 		}
-		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
-			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") && !strings.HasPrefix(msg.URL, "https://twitter.com/") &&
+			!strings.HasPrefix(msg.URL, "https://plus.google.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
+			if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil {
+				log.Warn("Failed to send URL error to client", "err", err)
+				return
+			}
 			continue
 		}
 		if msg.Tier >= uint(*tiersFlag) {
-			websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"})
+			if err = sendError(conn, errors.New("Invalid funding tier requested")); err != nil {
+				log.Warn("Failed to send tier error to client", "err", err)
+				return
+			}
 			continue
 		}
-		log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier)
+		log.Info("Faucet funds requested", "url", msg.URL, "tier", msg.Tier)
 
 		// If captcha verifications are enabled, make sure we're not dealing with a robot
 		if *captchaToken != "" {
@@ -362,7 +399,10 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 
 			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send captcha post error to client", "err", err)
+					return
+				}
 				continue
 			}
 			var result struct {
@@ -372,74 +412,55 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			err = json.NewDecoder(res.Body).Decode(&result)
 			res.Body.Close()
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send captcha decode error to client", "err", err)
+					return
+				}
 				continue
 			}
 			if !result.Success {
 				log.Warn("Captcha verification failed", "err", string(result.Errors))
-				websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"})
+				if err = sendError(conn, errors.New("Beep-bop, you're a robot!")); err != nil {
+					log.Warn("Failed to send captcha failure to client", "err", err)
+					return
+				}
 				continue
 			}
 		}
-		// Retrieve the gist from the GitHub Gist APIs
-		parts := strings.Split(msg.URL, "/")
-		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
-		if *githubUser != "" {
-			req.SetBasicAuth(*githubUser, *githubToken)
+		// Retrieve the Ethereum address to fund, the requesting user and a profile picture
+		var (
+			username string
+			avatar   string
+			address  common.Address
+		)
+		switch {
+		case strings.HasPrefix(msg.URL, "https://gist.github.com/"):
+			username, avatar, address, err = authGitHub(msg.URL)
+		case strings.HasPrefix(msg.URL, "https://twitter.com/"):
+			username, avatar, address, err = authTwitter(msg.URL)
+		case strings.HasPrefix(msg.URL, "https://plus.google.com/"):
+			username, avatar, address, err = authGooglePlus(msg.URL)
+		case strings.HasPrefix(msg.URL, "https://www.facebook.com/"):
+			username, avatar, address, err = authFacebook(msg.URL)
+		default:
+			err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues")
 		}
-		res, err := http.DefaultClient.Do(req)
 		if err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
-			continue
-		}
-		var gist struct {
-			Owner struct {
-				Login string `json:"login"`
-			} `json:"owner"`
-			Files map[string]struct {
-				Content string `json:"content"`
-			} `json:"files"`
-		}
-		err = json.NewDecoder(res.Body).Decode(&gist)
-		res.Body.Close()
-		if err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
-			continue
-		}
-		if gist.Owner.Login == "" {
-			websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"})
-			continue
-		}
-		// Iterate over all the files and look for Ethereum addresses
-		var address common.Address
-		for _, file := range gist.Files {
-			content := strings.TrimSpace(file.Content)
-			if len(content) == 2+common.AddressLength*2 {
-				address = common.HexToAddress(content)
+			if err = sendError(conn, err); err != nil {
+				log.Warn("Failed to send prefix error to client", "err", err)
+				return
 			}
-		}
-		if address == (common.Address{}) {
-			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
 			continue
 		}
-		// Validate the user's existence since the API is unhelpful here
-		if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
-			continue
-		}
-		res.Body.Close()
+		log.Info("Faucet request valid", "url", msg.URL, "tier", msg.Tier, "user", username, "address", address)
 
-		if res.StatusCode != 200 {
-			websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"})
-			continue
-		}
 		// Ensure the user didn't request funds too recently
 		f.lock.Lock()
 		var (
 			fund    bool
 			timeout time.Time
 		)
-		if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) {
+		if timeout = f.timeouts[username]; time.Now().After(timeout) {
 			// User wasn't funded recently, create the funding transaction
 			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
 			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
@@ -448,33 +469,45 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil)
 			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
 				f.lock.Unlock()
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send transaction creation error to client", "err", err)
+					return
+				}
 				continue
 			}
 			// Submit the transaction and mark as funded if successful
 			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
 				f.lock.Unlock()
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send transaction transmission error to client", "err", err)
+					return
+				}
 				continue
 			}
 			f.reqs = append(f.reqs, &request{
-				Username: gist.Owner.Login,
-				Account:  address,
-				Time:     time.Now(),
-				Tx:       signed,
+				Avatar:  avatar,
+				Account: address,
+				Time:    time.Now(),
+				Tx:      signed,
 			})
-			f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			f.timeouts[username] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
 			fund = true
 		}
 		f.lock.Unlock()
 
 		// Send an error if too frequent funding, othewise a success
 		if !fund {
-			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))})
+			if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))); err != nil {
+				log.Warn("Failed to send funding error to client", "err", err)
+				return
+			}
 			continue
 		}
-		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		if err = sendSuccess(conn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil {
+			log.Warn("Failed to send funding success to client", "err", err)
+			return
+		}
 		select {
 		case f.update <- struct{}{}:
 		default:
@@ -497,11 +530,31 @@ func (f *faucet) loop() {
 		select {
 		case head := <-heads:
 			// New chain head arrived, query the current stats and stream to clients
-			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
-			balance = new(big.Int).Div(balance, ether)
+			var (
+				balance *big.Int
+				nonce   uint64
+				price   *big.Int
+				err     error
+			)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
+			if err == nil {
+				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
+				if err == nil {
+					price, err = f.client.SuggestGasPrice(ctx)
+				}
+			}
+			cancel()
 
-			price, _ := f.client.SuggestGasPrice(context.Background())
-			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+			// If querying the data failed, try for the next block
+			if err != nil {
+				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
+				continue
+			} else {
+				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
+			}
+			// Faucet state retrieved, update locally and send to clients
+			balance = new(big.Int).Div(balance, ether)
 
 			f.lock.Lock()
 			f.price, f.nonce = price, nonce
@@ -512,17 +565,17 @@ func (f *faucet) loop() {
 
 			f.lock.RLock()
 			for _, conn := range f.conns {
-				if err := websocket.JSON.Send(conn, map[string]interface{}{
+				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
 					"peers":    f.stack.Server().PeerCount(),
 					"requests": f.reqs,
-				}); err != nil {
+				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)
 					conn.Close()
 					continue
 				}
-				if err := websocket.JSON.Send(conn, head); err != nil {
+				if err := send(conn, head, time.Second); err != nil {
 					log.Warn("Failed to send header to client", "err", err)
 					conn.Close()
 				}
@@ -533,7 +586,7 @@ func (f *faucet) loop() {
 			// Pending requests updated, stream to clients
 			f.lock.RLock()
 			for _, conn := range f.conns {
-				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+				if err := send(conn, map[string]interface{}{"requests": f.reqs}, time.Second); err != nil {
 					log.Warn("Failed to send requests to client", "err", err)
 					conn.Close()
 				}
@@ -542,3 +595,184 @@ func (f *faucet) loop() {
 		}
 	}
 }
+
+// sends transmits a data packet to the remote end of the websocket, but also
+// setting a write deadline to prevent waiting forever on the node.
+func send(conn *websocket.Conn, value interface{}, timeout time.Duration) error {
+	if timeout == 0 {
+		timeout = 60 * time.Second
+	}
+	conn.SetWriteDeadline(time.Now().Add(timeout))
+	return websocket.JSON.Send(conn, value)
+}
+
+// sendError transmits an error to the remote end of the websocket, also setting
+// the write deadline to 1 second to prevent waiting forever.
+func sendError(conn *websocket.Conn, err error) error {
+	return send(conn, map[string]string{"error": err.Error()}, time.Second)
+}
+
+// sendSuccess transmits a success message to the remote end of the websocket, also
+// setting the write deadline to 1 second to prevent waiting forever.
+func sendSuccess(conn *websocket.Conn, msg string) error {
+	return send(conn, map[string]string{"success": msg}, time.Second)
+}
+
+// authGitHub tries to authenticate a faucet request using GitHub gists, returning
+// the username, avatar URL and Ethereum address to fund on success.
+func authGitHub(url string) (string, string, common.Address, error) {
+	// Retrieve the gist from the GitHub Gist APIs
+	parts := strings.Split(url, "/")
+	req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+	if *githubUser != "" {
+		req.SetBasicAuth(*githubUser, *githubToken)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	var gist struct {
+		Owner struct {
+			Login string `json:"login"`
+		} `json:"owner"`
+		Files map[string]struct {
+			Content string `json:"content"`
+		} `json:"files"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&gist)
+	res.Body.Close()
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	if gist.Owner.Login == "" {
+		return "", "", common.Address{}, errors.New("Anonymous Gists not allowed")
+	}
+	// Iterate over all the files and look for Ethereum addresses
+	var address common.Address
+	for _, file := range gist.Files {
+		content := strings.TrimSpace(file.Content)
+		if len(content) == 2+common.AddressLength*2 {
+			address = common.HexToAddress(content)
+		}
+	}
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	// Validate the user's existence since the API is unhelpful here
+	if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
+		return "", "", common.Address{}, err
+	}
+	res.Body.Close()
+
+	if res.StatusCode != 200 {
+		return "", "", common.Address{}, errors.New("Invalid user... boom!")
+	}
+	// Everything passed validation, return the gathered infos
+	return gist.Owner.Login + "@github", fmt.Sprintf("https://github.com/%s.png?size=64", gist.Owner.Login), address, nil
+}
+
+// authTwitter tries to authenticate a faucet request using Twitter posts, returning
+// the username, avatar URL and Ethereum address to fund on success.
+func authTwitter(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "status" {
+		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Twitter's API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	reader, err := zlib.NewReader(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@twitter", avatar, address, nil
+}
+
+// authGooglePlus tries to authenticate a faucet request using GooglePlus posts,
+// returning the username, avatar URL and Ethereum address to fund on success.
+func authGooglePlus(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "posts" {
+		return "", "", common.Address{}, errors.New("Invalid Google+ post URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Google's API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+googleusercontent.com[^\"]+photo.jpg)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@google+", avatar, address, nil
+}
+
+// authFacebook tries to authenticate a faucet request using Facebook posts,
+// returning the username, avatar URL and Ethereum address to fund on success.
+func authFacebook(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "posts" {
+		return "", "", common.Address{}, errors.New("Invalid Facebook post URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Facebook's Graph API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+fbcdn.net[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@facebook", avatar, address, nil
+}

cmd/faucet/faucet.html

@@ -5,7 +5,7 @@
 		<meta http-equiv="X-UA-Compatible" content="IE=edge">
 		<meta name="viewport" content="width=device-width, initial-scale=1">
 
-		<title>{{.Network}}: GitHub Faucet</title>
+		<title>{{.Network}}: Authenticated Faucet</title>
 
 		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
 		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
@@ -43,13 +43,13 @@
 			<div class="container">
 				<div class="row" style="margin-bottom: 16px;">
 					<div class="col-lg-12">
-						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} Authenticated Faucet</h1>
 					</div>
 				</div>
 				<div class="row">
 					<div class="col-lg-8 col-lg-offset-2">
 						<div class="input-group">
-							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<input id="url" name="url" type="text" class="form-control" placeholder="Social network URL containing your Ethereum address...">
 							<span class="input-group-btn">
 								<button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Give me Ether	<i class="fa fa-caret-down" aria-hidden="true"></i></button>
 				        <ul class="dropdown-menu dropdown-menu-right">{{range $idx, $amount := .Amounts}}
@@ -80,8 +80,21 @@
 				<div class="row" style="margin-top: 32px;">
 					<div class="col-lg-12">
 						<h3>How does this work?</h3>
-						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.</p>
-						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to certain common 3rd party accounts. Anyone having a GitHub, Twitter, Google+ or Facebook account may request funds within the permitted limits.</p>
+						<dl class="dl-horizontal">
+						  <dt style="width: auto; margin-left: 40px;"><i class="fa fa-github-alt" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via GitHub, create a <a href="https://gist.github.com/" target="_about:blank">gist</a> with your Ethereum address embedded into the content (the file name doesn't matter).<br/>Copy-paste the gists URL into the above input box and fire away!</dd>
+
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-twitter" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Twitter, make a <a href="https://twitter.com/intent/tweet?text=Requesting%20faucet%20funds%20into%200x0000000000000000000000000000000000000000%20on%20the%20%23{{.Network}}%20%23Ethereum%20test%20network." target="_about:blank">tweet</a> with your Ethereum address pasted into the contents (surrounding text doesn't matter).<br/>Copy-paste the <a href="https://support.twitter.com/articles/80586" target="_about:blank">tweets URL</a> into the above input box and fire away!</dd>
+
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-google-plus-official" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Google Plus, publish a new <strong>public</strong> post with your Ethereum address embedded into the content (surrounding text doesn't matter).<br/>Copy-paste the posts URL into the above input box and fire away!</dd>
+
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-facebook" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Facebook, publish a new <strong>public</strong> post with your Ethereum address embedded into the content (surrounding text doesn't matter).<br/>Copy-paste the <a href="https://www.facebook.com/help/community/question/?id=282662498552845" target="_about:blank">posts URL</a> into the above input box and fire away!</dd>
+						</dl>
+						<p>You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
 						{{if .Recaptcha}}<em>The faucet is running invisible reCaptcha protection against bots.</em>{{end}}
 					</div>
 				</div>
@@ -93,10 +106,22 @@
 			var attempt = 0;
 			var server;
 			var tier = 0;
+			var requests = [];
 
+			// Define a function that creates closures to drop old requests
+			var dropper = function(hash) {
+				return function() {
+					for (var i=0; i<requests.length; i++) {
+						if (requests[i].tx.hash == hash) {
+							requests.splice(i, 1);
+							break;
+						}
+					}
+				}
+			};
 			// Define the function that submits a gist url to the server
 			var submit = function({{if .Recaptcha}}captcha{{end}}) {
-				server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
+				server.send(JSON.stringify({url: $("#url")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
 				grecaptcha.reset();{{end}}
 			};
 			// Define a method to reconnect upon server loss
@@ -127,21 +152,85 @@
 						$("#block").text(parseInt(msg.number, 16));
 					}
 					if (msg.error !== undefined) {
-						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+						noty({layout: 'topCenter', text: msg.error, type: 'error', timeout: 5000, progressBar: true});
 					}
 					if (msg.success !== undefined) {
-						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+						noty({layout: 'topCenter', text: msg.success, type: 'success', timeout: 5000, progressBar: true});
 					}
 					if (msg.requests !== undefined && msg.requests !== null) {
+						// Mark all previous requests missing as done
+						for (var i=0; i<requests.length; i++) {
+							if (msg.requests.length > 0 && msg.requests[0].tx.hash == requests[i].tx.hash) {
+								break;
+							}
+							if (requests[i].time != "") {
+								requests[i].time = "";
+								setTimeout(dropper(requests[i].tx.hash), 3000);
+							}
+						}
+						// Append any new requests into our local collection
+						var common = -1;
+						if (requests.length > 0) {
+							for (var i=0; i<msg.requests.length; i++) {
+								if (requests[requests.length-1].tx.hash == msg.requests[i].tx.hash) {
+									common = i;
+									break;
+								}
+							}
+						}
+						for (var i=common+1; i<msg.requests.length; i++) {
+							requests.push(msg.requests[i]);
+						}
+						// Iterate over our entire local collection and re-render the funding table
 						var content = "";
-						for (var i=0; i<msg.requests.length; i++) {
-							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+						for (var i=0; i<requests.length; i++) {
+							var done    = requests[i].time == "";
+							var elapsed = moment().unix()-moment(requests[i].time).unix();
+
+							content += "<tr id='" + requests[i].tx.hash + "'>";
+							content += "  <td><div style=\"background: url('" + requests[i].avatar + "'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td>";
+							content += "  <td><pre>" + requests[i].account + "</pre></td>";
+							content += "  <td style=\"width: 100%; text-align: center; vertical-align: middle;\">";
+							if (done) {
+								content += "    funded";
+							} else {
+								content += "    <span id='time-" + i + "' class='timer'>" + moment.duration(-elapsed, 'seconds').humanize(true) + "</span>";
+							}
+							content += "    <div class='progress' style='height: 4px; margin: 0;'>";
+							if (done) {
+								content += "      <div class='progress-bar progress-bar-success' role='progressbar' aria-valuenow='30' style='width:100%;'></div>";
+							} else if (elapsed > 30) {
+								content += "      <div class='progress-bar progress-bar-danger progress-bar-striped active' role='progressbar' aria-valuenow='30' style='width:100%;'></div>";
+							} else {
+								content += "      <div class='progress-bar progress-bar-striped active' role='progressbar' aria-valuenow='" + elapsed + "' style='width:" + (elapsed * 100 / 30) + "%;'></div>";
+							}
+							content += "    </div>";
+							content += "  </td>";
+							content += "</tr>";
 						}
 						$("#requests").html("<tbody>" + content + "</tbody>");
 					}
 				}
 				server.onclose = function() { setTimeout(reconnect, 3000); };
 			}
+			// Start a UI updater to push the progress bars forward until they are done
+			setInterval(function() {
+				$('.progress-bar').each(function() {
+					var progress = Number($(this).attr('aria-valuenow')) + 1;
+					if (progress < 30) {
+						$(this).attr('aria-valuenow', progress);
+						$(this).css('width', (progress * 100 / 30) + '%');
+					} else if (progress == 30) {
+						$(this).css('width', '100%');
+						$(this).addClass("progress-bar-danger");
+					}
+				})
+				$('.timer').each(function() {
+					var index = Number($(this).attr('id').substring(5));
+					$(this).html(moment.duration(moment(requests[index].time).unix()-moment().unix(), 'seconds').humanize(true));
+				})
+			}, 1000);
+
 			// Establish a websocket connection to the API server
 			reconnect();
 		</script>{{if .Recaptcha}}

cmd/geth/accountcmd.go

@@ -291,15 +291,28 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack, _ := makeConfigNode(ctx)
+	cfg := gethConfig{Node: defaultNodeConfig()}
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	scryptN, scryptP, keydir, err := cfg.Node.AccountConfig()
+
+	if err != nil {
+		utils.Fatalf("Failed to read configuration: %v", err)
+	}
+
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
-	account, err := ks.NewAccount(password)
+	address, err := keystore.StoreKey(keydir, password, scryptN, scryptP)
+
 	if err != nil {
 		utils.Fatalf("Failed to create account: %v", err)
 	}
-	fmt.Printf("Address: {%x}\n", account.Address)
+	fmt.Printf("Address: {%x}\n", address)
 	return nil
 }
 

cmd/geth/accountcmd_test.go

@@ -134,7 +134,7 @@ Fatal: could not decrypt key with given passphrase
 func TestUnlockFlag(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
 	geth.Expect(`
@@ -158,7 +158,7 @@ Passphrase: {{.InputLine "foobar"}}
 func TestUnlockFlagWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer geth.ExpectExit()
 	geth.Expect(`
@@ -177,7 +177,7 @@ Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could
 func TestUnlockFlagMultiIndex(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "0,2",
 		"js", "testdata/empty.js")
 	geth.Expect(`
@@ -204,7 +204,7 @@ Passphrase: {{.InputLine "foobar"}}
 func TestUnlockFlagPasswordFile(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--password", "testdata/passwords.txt", "--unlock", "0,2",
 		"js", "testdata/empty.js")
 	geth.ExpectExit()
@@ -224,7 +224,7 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
 	defer geth.ExpectExit()
 	geth.Expect(`
@@ -235,7 +235,7 @@ Fatal: Failed to unlock account 0 (could not decrypt key with given passphrase)
 func TestUnlockFlagAmbiguous(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
+		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
 	defer geth.ExpectExit()
@@ -273,7 +273,7 @@ In order to avoid this warning, you need to remove the following duplicate key f
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
+		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer geth.ExpectExit()
 

cmd/geth/chaincmd.go

@@ -31,7 +31,9 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -71,7 +73,7 @@ It expects the genesis file as argument.`,
 The import command imports blocks from an RLP-encoded form. The form can be one file
 with several RLP-encoded blocks, or several files can be used.
 
-If only one file is used, import error will result in failure. If several files are used, 
+If only one file is used, import error will result in failure. If several files are used,
 processing will proceed even if an individual RLP-file import failure occurs.`,
 	}
 	exportCommand = cli.Command{
@@ -90,6 +92,23 @@ Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
 if already existing.`,
+	}
+	copydbCommand = cli.Command{
+		Action:    utils.MigrateFlags(copyDb),
+		Name:      "copydb",
+		Usage:     "Create a local chain from a target chaindata folder",
+		ArgsUsage: "<sourceChaindataDir>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.SyncModeFlag,
+			utils.FakePoWFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+		Description: `
+The first argument must be the directory containing the blockchain to download from`,
 	}
 	removedbCommand = cli.Command{
 		Action:    utils.MigrateFlags(removeDB),
@@ -268,6 +287,54 @@ func exportChain(ctx *cli.Context) error {
 	return nil
 }
 
+func copyDb(ctx *cli.Context) error {
+	// Ensure we have a source chain directory to copy
+	if len(ctx.Args()) != 1 {
+		utils.Fatalf("Source chaindata directory path argument missing")
+	}
+	// Initialize a new chain for the running node to sync into
+	stack := makeFullNode(ctx)
+	chain, chainDb := utils.MakeChain(ctx, stack)
+
+	syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
+	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
+
+	// Create a source peer to satisfy downloader requests from
+	db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256)
+	if err != nil {
+		return err
+	}
+	hc, err := core.NewHeaderChain(db, chain.Config(), chain.Engine(), func() bool { return false })
+	if err != nil {
+		return err
+	}
+	peer := downloader.NewFakePeer("local", db, hc, dl)
+	if err = dl.RegisterPeer("local", 63, peer); err != nil {
+		return err
+	}
+	// Synchronise with the simulated peer
+	start := time.Now()
+
+	currentHeader := hc.CurrentHeader()
+	if err = dl.Synchronise("local", currentHeader.Hash(), hc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()), syncmode); err != nil {
+		return err
+	}
+	for dl.Synchronising() {
+		time.Sleep(10 * time.Millisecond)
+	}
+	fmt.Printf("Database copy done in %v\n", time.Since(start))
+
+	// Compact the entire database to remove any sync overhead
+	start = time.Now()
+	fmt.Println("Compacting entire database...")
+	if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil {
+		utils.Fatalf("Compaction failed: %v", err)
+	}
+	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
+
+	return nil
+}
+
 func removeDB(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 

cmd/geth/config.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/contracts/release"
+	"github.com/ethereum/go-ethereum/dashboard"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
@@ -76,10 +77,11 @@ type ethstatsConfig struct {
 }
 
 type gethConfig struct {
-	Eth      eth.Config
-	Shh      whisper.Config
-	Node     node.Config
-	Ethstats ethstatsConfig
+	Eth       eth.Config
+	Shh       whisper.Config
+	Node      node.Config
+	Ethstats  ethstatsConfig
+	Dashboard dashboard.Config
 }
 
 func loadConfig(file string, cfg *gethConfig) error {
@@ -110,9 +112,10 @@ func defaultNodeConfig() node.Config {
 func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	// Load defaults.
 	cfg := gethConfig{
-		Eth:  eth.DefaultConfig,
-		Shh:  whisper.DefaultConfig,
-		Node: defaultNodeConfig(),
+		Eth:       eth.DefaultConfig,
+		Shh:       whisper.DefaultConfig,
+		Node:      defaultNodeConfig(),
+		Dashboard: dashboard.DefaultConfig,
 	}
 
 	// Load config file.
@@ -134,6 +137,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	}
 
 	utils.SetShhConfig(ctx, stack, &cfg.Shh)
+	utils.SetDashboardConfig(ctx, &cfg.Dashboard)
 
 	return stack, cfg
 }
@@ -153,9 +157,12 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 
 	utils.RegisterEthService(stack, &cfg.Eth)
 
+	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
+		utils.RegisterDashboardService(stack, &cfg.Dashboard)
+	}
 	// Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode
 	shhEnabled := enableWhisper(ctx)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DeveloperFlag.Name)
 	if shhEnabled || shhAutoEnabled {
 		if ctx.GlobalIsSet(utils.WhisperMaxMessageSizeFlag.Name) {
 			cfg.Shh.MaxMessageSize = uint32(ctx.Int(utils.WhisperMaxMessageSizeFlag.Name))

cmd/geth/main.go

@@ -61,6 +61,11 @@ var (
 		utils.DataDirFlag,
 		utils.KeyStoreDirFlag,
 		utils.NoUSBFlag,
+		utils.DashboardEnabledFlag,
+		utils.DashboardAddrFlag,
+		utils.DashboardPortFlag,
+		utils.DashboardRefreshFlag,
+		utils.DashboardAssetsFlag,
 		utils.EthashCacheDirFlag,
 		utils.EthashCachesInMemoryFlag,
 		utils.EthashCachesOnDiskFlag,
@@ -99,7 +104,8 @@ var (
 		utils.NetrestrictFlag,
 		utils.NodeKeyFileFlag,
 		utils.NodeKeyHexFlag,
-		utils.DevModeFlag,
+		utils.DeveloperFlag,
+		utils.DeveloperPeriodFlag,
 		utils.TestnetFlag,
 		utils.RinkebyFlag,
 		utils.VMEnableDebugFlag,
@@ -146,6 +152,7 @@ func init() {
 		initCommand,
 		importCommand,
 		exportCommand,
+		copydbCommand,
 		removedbCommand,
 		dumpCommand,
 		// See monitorcmd.go:
@@ -269,7 +276,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		}
 	}()
 	// Start auxiliary services if enabled
-	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
+	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
 		// Mining only makes sense if a full Ethereum node is running
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {

cmd/geth/usage.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"gopkg.in/urfave/cli.v1"
+	"strings"
 )
 
 // AppHelpTemplate is the test template for the default, global app help topic.
@@ -72,7 +73,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetworkIdFlag,
 			utils.TestnetFlag,
 			utils.RinkebyFlag,
-			utils.DevModeFlag,
 			utils.SyncModeFlag,
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
@@ -81,6 +81,12 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.LightKDFFlag,
 		},
 	},
+	{Name: "DEVELOPER CHAIN",
+		Flags: []cli.Flag{
+			utils.DeveloperFlag,
+			utils.DeveloperPeriodFlag,
+		},
+	},
 	{
 		Name: "ETHASH",
 		Flags: []cli.Flag{
@@ -92,6 +98,16 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.EthashDatasetsOnDiskFlag,
 		},
 	},
+	//{
+	//	Name: "DASHBOARD",
+	//	Flags: []cli.Flag{
+	//		utils.DashboardEnabledFlag,
+	//		utils.DashboardAddrFlag,
+	//		utils.DashboardPortFlag,
+	//		utils.DashboardRefreshFlag,
+	//		utils.DashboardAssetsFlag,
+	//	},
+	//},
 	{
 		Name: "TRANSACTION POOL",
 		Flags: []cli.Flag{
@@ -263,6 +279,9 @@ func init() {
 			uncategorized := []cli.Flag{}
 			for _, flag := range data.(*cli.App).Flags {
 				if _, ok := categorized[flag.String()]; !ok {
+					if strings.HasPrefix(flag.GetName(), "dashboard") {
+						continue
+					}
 					uncategorized = append(uncategorized, flag)
 				}
 			}

cmd/puppeth/module_faucet.go

@@ -133,7 +133,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
-	files[filepath.Join(workdir, "genesis.json")] = []byte(config.node.genesis)
+	files[filepath.Join(workdir, "genesis.json")] = config.node.genesis
 	files[filepath.Join(workdir, "account.json")] = []byte(config.node.keyJSON)
 	files[filepath.Join(workdir, "account.pass")] = []byte(config.node.keyPass)
 

cmd/puppeth/module_node.go

@@ -128,7 +128,7 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
 	//genesisfile, _ := json.MarshalIndent(config.genesis, "", "  ")
-	files[filepath.Join(workdir, "genesis.json")] = []byte(config.genesis)
+	files[filepath.Join(workdir, "genesis.json")] = config.genesis
 
 	if config.keyJSON != "" {
 		files[filepath.Join(workdir, "signer.json")] = []byte(config.keyJSON)

cmd/puppeth/ssh.go

@@ -27,7 +27,6 @@ import (
 	"os/user"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/ethereum/go-ethereum/log"
 	"golang.org/x/crypto/ssh"
@@ -78,14 +77,25 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 	} else {
 		key, err := ssh.ParsePrivateKey(buf)
 		if err != nil {
-			log.Warn("Bad SSH key, falling back to passwords", "path", path, "err", err)
+			fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path)
+			blob, err := terminal.ReadPassword(int(os.Stdin.Fd()))
+			fmt.Println()
+			if err != nil {
+				log.Warn("Couldn't read password", "err", err)
+			}
+			key, err := ssh.ParsePrivateKeyWithPassphrase(buf, blob)
+			if err != nil {
+				log.Warn("Failed to decrypt SSH key, falling back to passwords", "path", path, "err", err)
+			} else {
+				auths = append(auths, ssh.PublicKeys(key))
+			}
 		} else {
 			auths = append(auths, ssh.PublicKeys(key))
 		}
 	}
 	auths = append(auths, ssh.PasswordCallback(func() (string, error) {
 		fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server)
-		blob, err := terminal.ReadPassword(int(syscall.Stdin))
+		blob, err := terminal.ReadPassword(int(os.Stdin.Fd()))
 
 		fmt.Println()
 		return string(blob), err

cmd/puppeth/wizard.go

@@ -28,7 +28,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"syscall"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -161,6 +160,28 @@ func (w *wizard) readDefaultInt(def int) int {
 	}
 }
 
+// readDefaultBigInt reads a single line from stdin, trimming if from spaces,
+// enforcing it to parse into a big integer. If an empty line is entered, the
+// default value is returned.
+func (w *wizard) readDefaultBigInt(def *big.Int) *big.Int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, ok := new(big.Int).SetString(text, 0)
+		if !ok {
+			log.Error("Invalid input, expected big integer")
+			continue
+		}
+		return val
+	}
+}
+
 /*
 // readFloat reads a single line from stdin, trimming if from spaces, enforcing it
 // to parse into a float.
@@ -209,7 +230,7 @@ func (w *wizard) readDefaultFloat(def float64) float64 {
 // line and returns it. The input will not be echoed.
 func (w *wizard) readPassword() string {
 	fmt.Printf("> ")
-	text, err := terminal.ReadPassword(int(syscall.Stdin))
+	text, err := terminal.ReadPassword(int(os.Stdin.Fd()))
 	if err != nil {
 		log.Crit("Failed to read password", "err", err)
 	}
@@ -280,8 +301,10 @@ func (w *wizard) readJSON() string {
 }
 
 // readIPAddress reads a single line from stdin, trimming if from spaces and
-// converts it to a network IP address.
-func (w *wizard) readIPAddress() net.IP {
+// returning it if it's convertible to an IP address. The reason for keeping
+// the user input format instead of returning a Go net.IP is to match with
+// weird formats used by ethstats, which compares IPs textually, not by value.
+func (w *wizard) readIPAddress() string {
 	for {
 		// Read the IP address from the user
 		fmt.Printf("> ")
@@ -290,14 +313,13 @@ func (w *wizard) readIPAddress() net.IP {
 			log.Crit("Failed to read user input", "err", err)
 		}
 		if text = strings.TrimSpace(text); text == "" {
-			return nil
+			return ""
 		}
 		// Make sure it looks ok and return it if so
-		ip := net.ParseIP(text)
-		if ip == nil {
+		if ip := net.ParseIP(text); ip == nil {
 			log.Error("Invalid IP address, please retry")
 			continue
 		}
-		return ip
+		return text
 	}
 }

cmd/puppeth/wizard_ethstats.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/log"
 )
@@ -64,17 +65,37 @@ func (w *wizard) deployEthstats() {
 	fmt.Println()
 	fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned)
 	if w.readDefaultString("y") != "y" {
-		infos.banned = nil
-
+		// The user might want to clear the entire list, although generally probably not
 		fmt.Println()
-		fmt.Println("Which IP addresses should be blacklisted?")
+		fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n")
+		if w.readDefaultString("n") != "n" {
+			infos.banned = nil
+		}
+		// Offer the user to explicitly add/remove certain IP addresses
+		fmt.Println()
+		fmt.Println("Which additional IP addresses should be blacklisted?")
 		for {
-			if ip := w.readIPAddress(); ip != nil {
-				infos.banned = append(infos.banned, ip.String())
+			if ip := w.readIPAddress(); ip != "" {
+				infos.banned = append(infos.banned, ip)
 				continue
 			}
 			break
 		}
+		fmt.Println()
+		fmt.Println("Which IP addresses should not be blacklisted?")
+		for {
+			if ip := w.readIPAddress(); ip != "" {
+				for i, addr := range infos.banned {
+					if ip == addr {
+						infos.banned = append(infos.banned[:i], infos.banned[i+1:]...)
+						break
+					}
+				}
+				continue
+			}
+			break
+		}
+		sort.Strings(infos.banned)
 	}
 	// Try to deploy the ethstats server on the host
 	trusted := make([]string, 0, len(w.servers))

cmd/puppeth/wizard_genesis.go

@@ -18,7 +18,9 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/big"
 	"math/rand"
 	"time"
@@ -135,3 +137,53 @@ func (w *wizard) makeGenesis() {
 	// All done, store the genesis and flush to disk
 	w.conf.genesis = genesis
 }
+
+// manageGenesis permits the modification of chain configuration parameters in
+// a genesis config and the export of the entire genesis spec.
+func (w *wizard) manageGenesis() {
+	// Figure out whether to modify or export the genesis
+	fmt.Println()
+	fmt.Println(" 1. Modify existing fork rules")
+	fmt.Println(" 2. Export genesis configuration")
+
+	choice := w.read()
+	switch {
+	case choice == "1":
+		// Fork rule updating requested, iterate over each fork
+		fmt.Println()
+		fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.genesis.Config.HomesteadBlock)
+		w.conf.genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.genesis.Config.HomesteadBlock)
+
+		fmt.Println()
+		fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP150Block)
+		w.conf.genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP150Block)
+
+		fmt.Println()
+		fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP155Block)
+		w.conf.genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP155Block)
+
+		fmt.Println()
+		fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP158Block)
+		w.conf.genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP158Block)
+
+		fmt.Println()
+		fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.genesis.Config.ByzantiumBlock)
+		w.conf.genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.genesis.Config.ByzantiumBlock)
+
+		out, _ := json.MarshalIndent(w.conf.genesis.Config, "", "  ")
+		fmt.Printf("Chain configuration updated:\n\n%s\n", out)
+
+	case choice == "2":
+		// Save whatever genesis configuration we currently have
+		fmt.Println()
+		fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
+		out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
+		if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
+			log.Error("Failed to save genesis file", "err", err)
+		}
+		log.Info("Exported existing genesis block")
+
+	default:
+		log.Error("That's not something I can do")
+	}
+}

cmd/puppeth/wizard_intro.go

@@ -98,7 +98,7 @@ func (w *wizard) run() {
 		if w.conf.genesis == nil {
 			fmt.Println(" 2. Configure new genesis")
 		} else {
-			fmt.Println(" 2. Save existing genesis")
+			fmt.Println(" 2. Manage existing genesis")
 		}
 		if len(w.servers) == 0 {
 			fmt.Println(" 3. Track new remote server")
@@ -118,18 +118,10 @@ func (w *wizard) run() {
 			w.networkStats(false)
 
 		case choice == "2":
-			// If we don't have a genesis, make one
 			if w.conf.genesis == nil {
 				w.makeGenesis()
 			} else {
-				// Otherwise just save whatever we currently have
-				fmt.Println()
-				fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
-				out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
-				if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
-					log.Error("Failed to save genesis file", "err", err)
-				}
-				log.Info("Exported existing genesis block")
+				w.manageGenesis()
 			}
 		case choice == "3":
 			if len(w.servers) == 0 {

cmd/puppeth/wizard_netstats.go

@@ -129,7 +129,7 @@ func (w *wizard) networkStats(tips bool) {
 		}
 	}
 	// If a genesis block was found, load it into our configs
-	if protips.genesis != "" {
+	if protips.genesis != "" && w.conf.genesis == nil {
 		genesis := new(core.Genesis)
 		if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
 			log.Error("Failed to parse remote genesis", "err", err)

cmd/puppeth/wizard_network.go

@@ -71,7 +71,7 @@ func (w *wizard) makeServer() string {
 	fmt.Println()
 	fmt.Println("Please enter remote server's address:")
 
-	// Read and fial the server to ensure docker is present
+	// Read and dial the server to ensure docker is present
 	input := w.readString()
 
 	client, err := dial(input, nil)

cmd/rlpdump/main.go

@@ -51,7 +51,7 @@ func main() {
 	var r io.Reader
 	switch {
 	case *hexMode != "":
-		data, err := hex.DecodeString(*hexMode)
+		data, err := hex.DecodeString(strings.TrimPrefix(*hexMode, "0x"))
 		if err != nil {
 			die(err)
 		}

cmd/utils/flags.go

@@ -31,11 +31,14 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/dashboard"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/gasprice"
@@ -135,9 +138,13 @@ var (
 		Name:  "rinkeby",
 		Usage: "Rinkeby network: pre-configured proof-of-authority test network",
 	}
-	DevModeFlag = cli.BoolFlag{
+	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
-		Usage: "Developer mode: pre-configured private network with several debugging flags",
+		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
+	}
+	DeveloperPeriodFlag = cli.IntFlag{
+		Name:  "dev.period",
+		Usage: "Block period to use in developer mode (0 = mine only if transaction pending)",
 	}
 	IdentityFlag = cli.StringFlag{
 		Name:  "identity",
@@ -177,6 +184,31 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	// Dashboard settings
+	DashboardEnabledFlag = cli.BoolFlag{
+		Name:  "dashboard",
+		Usage: "Enable the dashboard",
+	}
+	DashboardAddrFlag = cli.StringFlag{
+		Name:  "dashboard.addr",
+		Usage: "Dashboard listening interface",
+		Value: dashboard.DefaultConfig.Host,
+	}
+	DashboardPortFlag = cli.IntFlag{
+		Name:  "dashboard.host",
+		Usage: "Dashboard listening port",
+		Value: dashboard.DefaultConfig.Port,
+	}
+	DashboardRefreshFlag = cli.DurationFlag{
+		Name:  "dashboard.refresh",
+		Usage: "Dashboard metrics collection refresh rate",
+		Value: dashboard.DefaultConfig.Refresh,
+	}
+	DashboardAssetsFlag = cli.StringFlag{
+		Name:  "dashboard.assets",
+		Usage: "Developer flag to serve the dashboard from the local file system",
+		Value: dashboard.DefaultConfig.Assets,
+	}
 	// Ethash settings
 	EthashCacheDirFlag = DirectoryFlag{
 		Name:  "ethash.cachedir",
@@ -794,7 +826,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 		cfg.NetRestrict = list
 	}
 
-	if ctx.GlobalBool(DevModeFlag.Name) {
+	if ctx.GlobalBool(DeveloperFlag.Name) {
 		// --dev mode can't use p2p networking.
 		cfg.MaxPeers = 0
 		cfg.ListenAddr = ":0"
@@ -815,8 +847,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	switch {
 	case ctx.GlobalIsSet(DataDirFlag.Name):
 		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
-	case ctx.GlobalBool(DevModeFlag.Name):
-		cfg.DataDir = filepath.Join(os.TempDir(), "ethereum_dev_mode")
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		cfg.DataDir = "" // unless explicitly requested, use memory databases
 	case ctx.GlobalBool(TestnetFlag.Name):
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
 	case ctx.GlobalBool(RinkebyFlag.Name):
@@ -922,7 +954,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	// Avoid conflicting network flags
-	checkExclusive(ctx, DevModeFlag, TestnetFlag, RinkebyFlag)
+	checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag)
 	checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag)
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -983,20 +1015,44 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 			cfg.NetworkId = 4
 		}
 		cfg.Genesis = core.DefaultRinkebyGenesisBlock()
-	case ctx.GlobalBool(DevModeFlag.Name):
-		cfg.Genesis = core.DevGenesisBlock()
-		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
-			cfg.GasPrice = new(big.Int)
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		// Create new developer account or reuse existing one
+		var (
+			developer accounts.Account
+			err       error
+		)
+		if accs := ks.Accounts(); len(accs) > 0 {
+			developer = ks.Accounts()[0]
+		} else {
+			developer, err = ks.NewAccount("")
+			if err != nil {
+				Fatalf("Failed to create developer account: %v", err)
+			}
 		}
-		cfg.PowTest = true
-	}
+		if err := ks.Unlock(developer, ""); err != nil {
+			Fatalf("Failed to unlock developer account: %v", err)
+		}
+		log.Info("Using developer account", "address", developer.Address)
 
+		cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address)
+		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
+			cfg.GasPrice = big.NewInt(1)
+		}
+	}
 	// TODO(fjl): move trie cache generations into config
 	if gen := ctx.GlobalInt(TrieCacheGenFlag.Name); gen > 0 {
 		state.MaxTrieCacheGen = uint16(gen)
 	}
 }
 
+// SetDashboardConfig applies dashboard related command line flags to the config.
+func SetDashboardConfig(ctx *cli.Context, cfg *dashboard.Config) {
+	cfg.Host = ctx.GlobalString(DashboardAddrFlag.Name)
+	cfg.Port = ctx.GlobalInt(DashboardPortFlag.Name)
+	cfg.Refresh = ctx.GlobalDuration(DashboardRefreshFlag.Name)
+	cfg.Assets = ctx.GlobalString(DashboardAssetsFlag.Name)
+}
+
 // RegisterEthService adds an Ethereum client to the stack.
 func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 	var err error
@@ -1019,6 +1075,13 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 	}
 }
 
+// RegisterDashboardService adds a dashboard to the stack.
+func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config) {
+	stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		return dashboard.New(cfg)
+	})
+}
+
 // RegisterShhService configures Whisper and adds it to the given node.
 func RegisterShhService(stack *node.Node, cfg *whisper.Config) {
 	if err := stack.Register(func(n *node.ServiceContext) (node.Service, error) {
@@ -1075,8 +1138,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 		genesis = core.DefaultTestnetGenesisBlock()
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		genesis = core.DefaultRinkebyGenesisBlock()
-	case ctx.GlobalBool(DevModeFlag.Name):
-		genesis = core.DevGenesisBlock()
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		Fatalf("Developer chains are ephemeral")
 	}
 	return genesis
 }
@@ -1086,17 +1149,22 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	var err error
 	chainDb = MakeChainDatabase(ctx, stack)
 
-	engine := ethash.NewFaker()
-	if !ctx.GlobalBool(FakePoWFlag.Name) {
-		engine = ethash.New(
-			stack.ResolvePath(eth.DefaultConfig.EthashCacheDir), eth.DefaultConfig.EthashCachesInMem, eth.DefaultConfig.EthashCachesOnDisk,
-			stack.ResolvePath(eth.DefaultConfig.EthashDatasetDir), eth.DefaultConfig.EthashDatasetsInMem, eth.DefaultConfig.EthashDatasetsOnDisk,
-		)
-	}
 	config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
 	if err != nil {
 		Fatalf("%v", err)
 	}
+	var engine consensus.Engine
+	if config.Clique != nil {
+		engine = clique.New(config.Clique, chainDb)
+	} else {
+		engine = ethash.NewFaker()
+		if !ctx.GlobalBool(FakePoWFlag.Name) {
+			engine = ethash.New(
+				stack.ResolvePath(eth.DefaultConfig.EthashCacheDir), eth.DefaultConfig.EthashCachesInMem, eth.DefaultConfig.EthashCachesOnDisk,
+				stack.ResolvePath(eth.DefaultConfig.EthashDatasetDir), eth.DefaultConfig.EthashDatasetsInMem, eth.DefaultConfig.EthashDatasetsOnDisk,
+			)
+		}
+	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
 	if err != nil {

common/hexutil/hexutil.go

@@ -53,9 +53,7 @@ var (
 
 type decError struct{ msg string }
 
-func (err decError) Error() string {
-	return string(err.msg)
-}
+func (err decError) Error() string { return err.msg }
 
 // Decode decodes a hex string with 0x prefix.
 func Decode(input string) ([]byte, error) {

common/hexutil/json.go

@@ -223,7 +223,7 @@ func (b *Uint64) UnmarshalText(input []byte) error {
 			return ErrSyntax
 		}
 		dec *= 16
-		dec += uint64(nib)
+		dec += nib
 	}
 	*b = Uint64(dec)
 	return nil

consensus/clique/clique.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -124,6 +125,11 @@ var (
 
 	// errUnauthorized is returned if a header is signed by a non-authorized entity.
 	errUnauthorized = errors.New("unauthorized")
+
+	// errWaitTransactions is returned if an empty block is attempted to be sealed
+	// on an instant chain (0 second period). It's important to refuse these as the
+	// block reward is zero, so an empty block just bloats the chain... fast.
+	errWaitTransactions = errors.New("waiting for transactions")
 )
 
 // SignerFn is a signer callback function to request a hash to be signed by a
@@ -210,9 +216,6 @@ func New(config *params.CliqueConfig, db ethdb.Database) *Clique {
 	if conf.Epoch == 0 {
 		conf.Epoch = epochLength
 	}
-	if conf.Period == 0 {
-		conf.Period = blockPeriod
-	}
 	// Allocate the snapshot caches and create the engine
 	recents, _ := lru.NewARC(inmemorySnapshots)
 	signatures, _ := lru.NewARC(inmemorySignatures)
@@ -313,6 +316,10 @@ func (c *Clique) verifyHeader(chain consensus.ChainReader, header *types.Header,
 			return errInvalidDifficulty
 		}
 	}
+	// If all checks passed, validate any special fields for hard forks
+	if err := misc.VerifyForkHashes(chain.Config(), header, false); err != nil {
+		return err
+	}
 	// All basic checks passed, verify cascading fields
 	return c.verifyCascadingFields(chain, header, parents)
 }
@@ -594,6 +601,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 	if number == 0 {
 		return nil, errUnknownBlock
 	}
+	// For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing)
+	if c.config.Period == 0 && len(block.Transactions()) == 0 {
+		return nil, errWaitTransactions
+	}
 	// Don't hold the signer fields for the entire sealing procedure
 	c.lock.RLock()
 	signer, signFn := c.signer, c.signFn

consensus/clique/snapshot_test.go

@@ -74,7 +74,7 @@ type testerChainReader struct {
 	db ethdb.Database
 }
 
-func (r *testerChainReader) Config() *params.ChainConfig                 { panic("not supported") }
+func (r *testerChainReader) Config() *params.ChainConfig                 { return params.AllCliqueProtocolChanges }
 func (r *testerChainReader) CurrentHeader() *types.Header                { panic("not supported") }
 func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
 func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block   { panic("not supported") }

consensus/ethash/algorithm.go

@@ -102,7 +102,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) {
 	header.Cap *= 4
 	cache := *(*[]byte)(unsafe.Pointer(&header))
 
-	// Calculate the number of thoretical rows (we'll store in one buffer nonetheless)
+	// Calculate the number of theoretical rows (we'll store in one buffer nonetheless)
 	size := uint64(len(cache))
 	rows := int(size) / hashBytes
 
@@ -187,7 +187,7 @@ func fnvHash(mix []uint32, data []uint32) {
 // generateDatasetItem combines data from 256 pseudorandomly selected cache nodes,
 // and hashes that to compute a single dataset node.
 func generateDatasetItem(cache []uint32, index uint32, keccak512 hasher) []byte {
-	// Calculate the number of thoretical rows (we use one buffer nonetheless)
+	// Calculate the number of theoretical rows (we use one buffer nonetheless)
 	rows := uint32(len(cache) / hashWords)
 
 	// Initialize the mix
@@ -287,7 +287,7 @@ func generateDataset(dest []uint32, epoch uint64, cache []uint32) {
 // hashimoto aggregates data from the full dataset in order to produce our final
 // value for a particular header hash and nonce.
 func hashimoto(hash []byte, nonce uint64, size uint64, lookup func(index uint32) []uint32) ([]byte, []byte) {
-	// Calculate the number of thoretical rows (we use one buffer nonetheless)
+	// Calculate the number of theoretical rows (we use one buffer nonetheless)
 	rows := uint32(size / mixBytes)
 
 	// Combine header+nonce into a 64 byte seed

consensus/ethash/algorithm_go1.8.go

@@ -31,7 +31,7 @@ func cacheSize(block uint64) uint64 {
 		return cacheSizes[epoch]
 	}
 	// No known cache size, calculate manually (sanity branch only)
-	size := uint64(cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes)
+	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
 	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * hashBytes
 	}
@@ -49,7 +49,7 @@ func datasetSize(block uint64) uint64 {
 		return datasetSizes[epoch]
 	}
 	// No known dataset size, calculate manually (sanity branch only)
-	size := uint64(datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes)
+	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
 	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * mixBytes
 	}

console/console_test.go

@@ -94,7 +94,7 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester {
 		t.Fatalf("failed to create node: %v", err)
 	}
 	ethConf := &eth.Config{
-		Genesis:   core.DevGenesisBlock(),
+		Genesis:   core.DeveloperGenesisBlock(15, common.Address{}),
 		Etherbase: common.HexToAddress(testAddress),
 		PowTest:   true,
 	}

contracts/chequebook/contract/chequebook.sol

@@ -1,7 +1,9 @@
-import "mortal";
+pragma solidity ^0.4.18;
+
+import "https://github.com/ethereum/solidity/std/mortal.sol";
 
 /// @title Chequebook for Ethereum micropayments
-/// @author Daniel A. Nagy <daniel@ethdev.com>
+/// @author Daniel A. Nagy <daniel@ethereum.org>
 contract chequebook is mortal {
     // Cumulative paid amount in wei to each beneficiary
     mapping (address => uint256) public sent;
@@ -21,26 +23,23 @@ contract chequebook is mortal {
         uint8 sig_v, bytes32 sig_r, bytes32 sig_s) {
         // Check if the cheque is old.
         // Only cheques that are more recent than the last cashed one are considered.
-        if(amount <= sent[beneficiary]) return;
+        require(amount > sent[beneficiary]);
         // Check the digital signature of the cheque.
-        bytes32 hash = sha3(address(this), beneficiary, amount);
-        if(owner != ecrecover(hash, sig_v, sig_r, sig_s)) return;
+        bytes32 hash = keccak256(address(this), beneficiary, amount);
+        require(owner == ecrecover(hash, sig_v, sig_r, sig_s));
         // Attempt sending the difference between the cumulative amount on the cheque
         // and the cumulative amount on the last cashed cheque to beneficiary.
         uint256 diff = amount - sent[beneficiary];
         if (diff <= this.balance) {
 	    // update the cumulative amount before sending
             sent[beneficiary] = amount;
-            if (!beneficiary.send(diff)) {
-                // Upon failure to execute send, revert everything
-                throw;
-            }
+            beneficiary.transfer(diff);
         } else {
             // Upon failure, punish owner for writing a bounced cheque.
             // owner.sendToDebtorsPrison();
             Overdraft(owner);
             // Compensate beneficiary.
-            suicide(beneficiary);
+            selfdestruct(beneficiary);
         }
     }
 }

core/blockchain.go

@@ -818,7 +818,12 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
 	// If the total difficulty is higher than our known, add it to the canonical chain
 	// Second clause in the if statement reduces the vulnerability to selfish mining.
 	// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
-	if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) {
+	reorg := externTd.Cmp(localTd) > 0
+	if !reorg && externTd.Cmp(localTd) == 0 {
+		// Split same-difficulty blocks by number, then at random
+		reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5)
+	}
+	if reorg {
 		// Reorganise the chain if the parent is not the head block
 		if block.ParentHash() != bc.currentBlock.Hash() {
 			if err := bc.reorg(bc.currentBlock, block); err != nil {

core/bloombits/matcher.go

@@ -18,6 +18,7 @@ package bloombits
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"math"
 	"sort"
@@ -56,10 +57,16 @@ type partialMatches struct {
 // Retrieval represents a request for retrieval task assignments for a given
 // bit with the given number of fetch elements, or a response for such a request.
 // It can also have the actual results set to be used as a delivery data struct.
+//
+// The contest and error fields are used by the light client to terminate matching
+// early if an error is enountered on some path of the pipeline.
 type Retrieval struct {
 	Bit      uint
 	Sections []uint64
 	Bitsets  [][]byte
+
+	Context context.Context
+	Error   error
 }
 
 // Matcher is a pipelined system of schedulers and logic matchers which perform
@@ -137,7 +144,7 @@ func (m *Matcher) addScheduler(idx uint) {
 // Start starts the matching process and returns a stream of bloom matches in
 // a given range of blocks. If there are no more matches in the range, the result
 // channel is closed.
-func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession, error) {
+func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) {
 	// Make sure we're not creating concurrent sessions
 	if atomic.SwapUint32(&m.running, 1) == 1 {
 		return nil, errors.New("matcher already running")
@@ -149,6 +156,7 @@ func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession
 		matcher: m,
 		quit:    make(chan struct{}),
 		kill:    make(chan struct{}),
+		ctx:     ctx,
 	}
 	for _, scheduler := range m.schedulers {
 		scheduler.reset()
@@ -184,10 +192,12 @@ func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession
 				}
 				// Iterate over all the blocks in the section and return the matching ones
 				for i := first; i <= last; i++ {
-					// Skip the entire byte if no matches are found inside
+					// Skip the entire byte if no matches are found inside (and we're processing an entire byte!)
 					next := res.bitset[(i-sectionStart)/8]
 					if next == 0 {
-						i += 7
+						if i%8 == 0 {
+							i += 7
+						}
 						continue
 					}
 					// Some bit it set, do the actual submatching
@@ -502,25 +512,34 @@ func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
 type MatcherSession struct {
 	matcher *Matcher
 
-	quit chan struct{} // Quit channel to request pipeline termination
-	kill chan struct{} // Term channel to signal non-graceful forced shutdown
+	closer sync.Once     // Sync object to ensure we only ever close once
+	quit   chan struct{} // Quit channel to request pipeline termination
+	kill   chan struct{} // Term channel to signal non-graceful forced shutdown
+
+	ctx context.Context // Context used by the light client to abort filtering
+	err atomic.Value    // Global error to track retrieval failures deep in the chain
+
 	pend sync.WaitGroup
 }
 
 // Close stops the matching process and waits for all subprocesses to terminate
 // before returning. The timeout may be used for graceful shutdown, allowing the
 // currently running retrievals to complete before this time.
-func (s *MatcherSession) Close(timeout time.Duration) {
-	// Bail out if the matcher is not running
-	select {
-	case <-s.quit:
-		return
-	default:
+func (s *MatcherSession) Close() {
+	s.closer.Do(func() {
+		// Signal termination and wait for all goroutines to tear down
+		close(s.quit)
+		time.AfterFunc(time.Second, func() { close(s.kill) })
+		s.pend.Wait()
+	})
+}
+
+// Error returns any failure encountered during the matching session.
+func (s *MatcherSession) Error() error {
+	if err := s.err.Load(); err != nil {
+		return err.(error)
 	}
-	// Signal termination and wait for all goroutines to tear down
-	close(s.quit)
-	time.AfterFunc(timeout, func() { close(s.kill) })
-	s.pend.Wait()
+	return nil
 }
 
 // AllocateRetrieval assigns a bloom bit index to a client process that can either
@@ -618,9 +637,13 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan
 
 		case mux <- request:
 			// Retrieval accepted, something must arrive before we're aborting
-			request <- &Retrieval{Bit: bit, Sections: sections}
+			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
 
 			result := <-request
+			if result.Error != nil {
+				s.err.Store(result.Error)
+				s.Close()
+			}
 			s.DeliverSections(result.Bit, result.Sections, result.Bitsets)
 		}
 	}

core/bloombits/matcher_test.go

@@ -17,6 +17,7 @@
 package bloombits
 
 import (
+	"context"
 	"math/rand"
 	"sync/atomic"
 	"testing"
@@ -30,14 +31,14 @@ const testSectionSize = 4096
 // Tests that wildcard filter rules (nil) can be specified and are handled well.
 func TestMatcherWildcards(t *testing.T) {
 	matcher := NewMatcher(testSectionSize, [][][]byte{
-		[][]byte{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
-		[][]byte{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
-		[][]byte{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
-		[][]byte{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
-		[][]byte{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
-		[][]byte{nil, nil},                                               // Wildcard combo, drop rule
-		[][]byte{},                                                       // Inited wildcard rule, drop rule
-		nil,                                                              // Proper wildcard rule, drop rule
+		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
+		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
+		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
+		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
+		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
+		{nil, nil},                                               // Wildcard combo, drop rule
+		{},                                                       // Inited wildcard rule, drop rule
+		nil,                                                      // Proper wildcard rule, drop rule
 	})
 	if len(matcher.filters) != 3 {
 		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
@@ -55,37 +56,52 @@ func TestMatcherWildcards(t *testing.T) {
 
 // Tests the matcher pipeline on a single continuous workflow without interrupts.
 func TestMatcherContinuous(t *testing.T) {
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
 }
 
 // Tests the matcher pipeline on a constantly interrupted and resumed work pattern
 // with the aim of ensuring data items are requested only once.
 func TestMatcherIntermittent(t *testing.T) {
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
 }
 
 // Tests the matcher pipeline on random input to hopefully catch anomalies.
 func TestMatcherRandom(t *testing.T) {
 	for i := 0; i < 10; i++ {
-		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
 	}
 }
 
+// Tests that the matcher can properly find matches if the starting block is
+// shifter from a multiple of 8. This is needed to cover an optimisation with
+// bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
+func TestMatcherShifted(t *testing.T) {
+	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
+	// start to get a potential zero byte in the matcher bitset.
+
+	// To keep the second bitset byte zero, the filter must only match for the first
+	// time in block 16, so doing an all-16 bit filter should suffice.
+
+	// To keep the starting block non divisible by 8, block number 9 is the first
+	// that would introduce a shift and not match block 0.
+	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
+}
+
 // Tests that matching on everything doesn't crash (special case internally).
 func TestWildcardMatcher(t *testing.T) {
-	testMatcherBothModes(t, nil, 10000, 0)
+	testMatcherBothModes(t, nil, 0, 10000, 0)
 }
 
 // makeRandomIndexes generates a random filter system, composed on multiple filter
-// criteria, each having one bloom list component for the address and arbitrarilly
+// criteria, each having one bloom list component for the address and arbitrarily
 // many topic bloom list components.
 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
 	res := make([][]bloomIndexes, len(lengths))
@@ -103,9 +119,9 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
 // testMatcherDiffBatches runs the given matches test in single-delivery and also
 // in batches delivery mode, verifying that all kinds of deliveries are handled
 // correctly withn.
-func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) {
-	singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1)
-	batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16)
+func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
+	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
+	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
 
 	if singleton != batched {
 		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
@@ -114,9 +130,9 @@ func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64
 
 // testMatcherBothModes runs the given matcher test in both continuous as well as
 // in intermittent mode, verifying that the request counts match each other.
-func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) {
-	continuous := testMatcher(t, filter, blocks, false, retrievals, 16)
-	intermittent := testMatcher(t, filter, blocks, true, retrievals, 16)
+func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
+	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
+	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
 
 	if continuous != intermittent {
 		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
@@ -125,7 +141,7 @@ func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64,
 
 // testMatcher is a generic tester to run the given matcher test and return the
 // number of requests made for cross validation between different modes.
-func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
+func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
 	// Create a new matcher an simulate our explicit random bitsets
 	matcher := NewMatcher(testSectionSize, nil)
 	matcher.filters = filter
@@ -144,14 +160,14 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 	quit := make(chan struct{})
 	matches := make(chan uint64, 16)
 
-	session, err := matcher.Start(0, blocks-1, matches)
+	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
 	if err != nil {
 		t.Fatalf("failed to stat matcher session: %v", err)
 	}
 	startRetrievers(session, quit, &requested, maxReqCount)
 
 	// Iterate over all the blocks and verify that the pipeline produces the correct matches
-	for i := uint64(0); i < blocks; i++ {
+	for i := start; i < blocks; i++ {
 		if expMatch3(filter, i) {
 			match, ok := <-matches
 			if !ok {
@@ -163,13 +179,13 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 			}
 			// If we're testing intermittent mode, abort and restart the pipeline
 			if intermittent {
-				session.Close(time.Second)
+				session.Close()
 				close(quit)
 
 				quit = make(chan struct{})
 				matches = make(chan uint64, 16)
 
-				session, err = matcher.Start(i+1, blocks-1, matches)
+				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
 				if err != nil {
 					t.Fatalf("failed to stat matcher session: %v", err)
 				}
@@ -183,7 +199,7 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
 	}
 	// Clean up the session and ensure we match the expected retrieval count
-	session.Close(time.Second)
+	session.Close()
 	close(quit)
 
 	if retrievals != 0 && requested != retrievals {

core/bloombits/scheduler_test.go

@@ -60,7 +60,7 @@ func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
 					req.section,                    // Requested data
 					req.section,                    // Duplicated data (ensure it doesn't double close anything)
 				}, [][]byte{
-					[]byte{},
+					{},
 					new(big.Int).SetUint64(req.section).Bytes(),
 					new(big.Int).SetUint64(req.section).Bytes(),
 				})

core/chain_indexer.go

@@ -36,7 +36,7 @@ import (
 type ChainIndexerBackend interface {
 	// Reset initiates the processing of a new chain segment, potentially terminating
 	// any partially completed operations (in case of a reorg).
-	Reset(section uint64)
+	Reset(section uint64, prevHead common.Hash) error
 
 	// Process crunches through the next header in the chain segment. The caller
 	// will ensure a sequential order of headers.
@@ -46,6 +46,15 @@ type ChainIndexerBackend interface {
 	Commit() error
 }
 
+// ChainIndexerChain interface is used for connecting the indexer to a blockchain
+type ChainIndexerChain interface {
+	// CurrentHeader retrieves the latest locally known header.
+	CurrentHeader() *types.Header
+
+	// SubscribeChainEvent subscribes to new head header notifications.
+	SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription
+}
+
 // ChainIndexer does a post-processing job for equally sized sections of the
 // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
 // connected to the blockchain through the event system by starting a
@@ -100,11 +109,27 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken
 	return c
 }
 
+// AddKnownSectionHead marks a new section head as known/processed if it is newer
+// than the already known best section head
+func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if section < c.storedSections {
+		return
+	}
+	c.setSectionHead(section, shead)
+	c.setValidSections(section + 1)
+}
+
 // Start creates a goroutine to feed chain head events into the indexer for
 // cascading background processing. Children do not need to be started, they
 // are notified about new events by their parents.
-func (c *ChainIndexer) Start(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
-	go c.eventLoop(currentHeader, chainEventer)
+func (c *ChainIndexer) Start(chain ChainIndexerChain) {
+	events := make(chan ChainEvent, 10)
+	sub := chain.SubscribeChainEvent(events)
+
+	go c.eventLoop(chain.CurrentHeader(), events, sub)
 }
 
 // Close tears down all goroutines belonging to the indexer and returns any error
@@ -147,12 +172,10 @@ func (c *ChainIndexer) Close() error {
 // eventLoop is a secondary - optional - event loop of the indexer which is only
 // started for the outermost indexer to push chain head events into a processing
 // queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
+func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) {
 	// Mark the chain indexer as active, requiring an additional teardown
 	atomic.StoreUint32(&c.active, 1)
 
-	events := make(chan ChainEvent, 10)
-	sub := chainEventer(events)
 	defer sub.Unsubscribe()
 
 	// Fire the initial new head event to start any outstanding processing
@@ -178,7 +201,11 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(
 			}
 			header := ev.Block.Header()
 			if header.ParentHash != prevHash {
-				c.newHead(FindCommonAncestor(c.chainDb, prevHeader, header).Number.Uint64(), true)
+				// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
+				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
+				if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
+					c.newHead(h.Number.Uint64(), true)
+				}
 			}
 			c.newHead(header.Number.Uint64(), false)
 
@@ -236,6 +263,7 @@ func (c *ChainIndexer) updateLoop() {
 		updating bool
 		updated  time.Time
 	)
+
 	for {
 		select {
 		case errc := <-c.quit:
@@ -259,7 +287,7 @@ func (c *ChainIndexer) updateLoop() {
 				section := c.storedSections
 				var oldHead common.Hash
 				if section > 0 {
-					oldHead = c.sectionHead(section - 1)
+					oldHead = c.SectionHead(section - 1)
 				}
 				// Process the newly defined section in the background
 				c.lock.Unlock()
@@ -270,7 +298,7 @@ func (c *ChainIndexer) updateLoop() {
 				c.lock.Lock()
 
 				// If processing succeeded and no reorgs occcurred, mark the section completed
-				if err == nil && oldHead == c.sectionHead(section-1) {
+				if err == nil && oldHead == c.SectionHead(section-1) {
 					c.setSectionHead(section, newHead)
 					c.setValidSections(section + 1)
 					if c.storedSections == c.knownSections && updating {
@@ -311,7 +339,11 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com
 	c.log.Trace("Processing new chain section", "section", section)
 
 	// Reset and partial processing
-	c.backend.Reset(section)
+
+	if err := c.backend.Reset(section, lastHead); err != nil {
+		c.setValidSections(0)
+		return common.Hash{}, err
+	}
 
 	for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
 		hash := GetCanonicalHash(c.chainDb, number)
@@ -341,7 +373,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	return c.storedSections, c.storedSections*c.sectionSize - 1, c.sectionHead(c.storedSections - 1)
+	return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
 }
 
 // AddChildIndexer adds a child ChainIndexer that can use the output of this one
@@ -381,9 +413,9 @@ func (c *ChainIndexer) setValidSections(sections uint64) {
 	c.storedSections = sections // needed if new > old
 }
 
-// sectionHead retrieves the last block hash of a processed section from the
+// SectionHead retrieves the last block hash of a processed section from the
 // index database.
-func (c *ChainIndexer) sectionHead(section uint64) common.Hash {
+func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
 	var data [8]byte
 	binary.BigEndian.PutUint64(data[:], section)
 

core/chain_indexer_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 )
@@ -208,9 +209,10 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
 	return b.stored * b.indexer.sectionSize
 }
 
-func (b *testChainIndexBackend) Reset(section uint64) {
+func (b *testChainIndexBackend) Reset(section uint64, prevHead common.Hash) error {
 	b.section = section
 	b.headerCnt = 0
+	return nil
 }
 
 func (b *testChainIndexBackend) Process(header *types.Header) {

core/chain_makers.go

@@ -235,7 +235,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) {
 	db, _ := ethdb.NewMemDatabase()
 	genesis := gspec.MustCommit(db)
 
-	blockchain, _ := NewBlockChain(db, params.AllProtocolChanges, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, params.AllEthashProtocolChanges, ethash.NewFaker(), vm.Config{})
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil

core/database_util.go

@@ -74,9 +74,9 @@ var (
 	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
 )
 
-// txLookupEntry is a positional metadata to help looking up the data content of
+// TxLookupEntry is a positional metadata to help looking up the data content of
 // a transaction or receipt given only its hash.
-type txLookupEntry struct {
+type TxLookupEntry struct {
 	BlockHash  common.Hash
 	BlockIndex uint64
 	Index      uint64
@@ -260,7 +260,7 @@ func GetTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64,
 		return common.Hash{}, 0, 0
 	}
 	// Parse and return the contents of the lookup entry
-	var entry txLookupEntry
+	var entry TxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
 		log.Error("Invalid lookup entry RLP", "hash", hash, "err", err)
 		return common.Hash{}, 0, 0
@@ -296,7 +296,7 @@ func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, co
 	if len(data) == 0 {
 		return nil, common.Hash{}, 0, 0
 	}
-	var entry txLookupEntry
+	var entry TxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
 		return nil, common.Hash{}, 0, 0
 	}
@@ -332,14 +332,13 @@ func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Has
 
 // GetBloomBits retrieves the compressed bloom bit vector belonging to the given
 // section and bit index from the.
-func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) []byte {
+func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) {
 	key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...)
 
 	binary.BigEndian.PutUint16(key[1:], uint16(bit))
 	binary.BigEndian.PutUint64(key[3:], section)
 
-	bits, _ := db.Get(key)
-	return bits
+	return db.Get(key)
 }
 
 // WriteCanonicalHash stores the canonical hash for the given block number.
@@ -465,7 +464,7 @@ func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receip
 func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error {
 	// Iterate over each transaction and encode its metadata
 	for i, tx := range block.Transactions() {
-		entry := txLookupEntry{
+		entry := TxLookupEntry{
 			BlockHash:  block.Hash(),
 			BlockIndex: block.NumberU64(),
 			Index:      uint64(i),

core/genesis.go

@@ -151,7 +151,7 @@ func (e *GenesisMismatchError) Error() string {
 // The returned chain configuration is never nil.
 func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) {
 	if genesis != nil && genesis.Config == nil {
-		return params.AllProtocolChanges, common.Hash{}, errGenesisNoConfig
+		return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
 
 	// Just commit the new block if there is no stored genesis block.
@@ -216,7 +216,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 	case ghash == params.TestnetGenesisHash:
 		return params.TestnetChainConfig
 	default:
-		return params.AllProtocolChanges
+		return params.AllEthashProtocolChanges
 	}
 }
 
@@ -285,7 +285,7 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
 	}
 	config := g.Config
 	if config == nil {
-		config = params.AllProtocolChanges
+		config = params.AllEthashProtocolChanges
 	}
 	return block, WriteChainConfig(db, block.Hash(), config)
 }
@@ -342,14 +342,30 @@ func DefaultRinkebyGenesisBlock() *Genesis {
 	}
 }
 
-// DevGenesisBlock returns the 'geth --dev' genesis block.
-func DevGenesisBlock() *Genesis {
+// DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must
+// be seeded with the
+func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis {
+	// Override the default period to the user requested one
+	config := *params.AllCliqueProtocolChanges
+	config.Clique.Period = period
+
+	// Assemble and return the genesis with the precompiles and faucet pre-funded
 	return &Genesis{
-		Config:     params.AllProtocolChanges,
-		Nonce:      42,
-		GasLimit:   4712388,
-		Difficulty: big.NewInt(131072),
-		Alloc:      decodePrealloc(devAllocData),
+		Config:     &config,
+		ExtraData:  append(append(make([]byte, 32), faucet[:]...), make([]byte, 65)...),
+		GasLimit:   6283185,
+		Difficulty: big.NewInt(1),
+		Alloc: map[common.Address]GenesisAccount{
+			common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover
+			common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256
+			common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD
+			common.BytesToAddress([]byte{4}): {Balance: big.NewInt(1)}, // Identity
+			common.BytesToAddress([]byte{5}): {Balance: big.NewInt(1)}, // ModExp
+			common.BytesToAddress([]byte{6}): {Balance: big.NewInt(1)}, // ECAdd
+			common.BytesToAddress([]byte{7}): {Balance: big.NewInt(1)}, // ECScalarMul
+			common.BytesToAddress([]byte{8}): {Balance: big.NewInt(1)}, // ECPairing
+			faucet: {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))},
+		},
 	}
 }
 

core/genesis_test.go

@@ -65,7 +65,7 @@ func TestSetupGenesis(t *testing.T) {
 				return SetupGenesisBlock(db, new(Genesis))
 			},
 			wantErr:    errGenesisNoConfig,
-			wantConfig: params.AllProtocolChanges,
+			wantConfig: params.AllEthashProtocolChanges,
 		},
 		{
 			name: "no block in DB, genesis == nil",

core/tx_journal.go

@@ -31,6 +31,15 @@ import (
 // into the journal, but no such file is currently open.
 var errNoActiveJournal = errors.New("no active journal")
 
+// devNull is a WriteCloser that just discards anything written into it. Its
+// goal is to allow the transaction journal to write into a fake journal when
+// loading transactions on startup without printing warnings due to no file
+// being readt for write.
+type devNull struct{}
+
+func (*devNull) Write(p []byte) (n int, err error) { return len(p), nil }
+func (*devNull) Close() error                      { return nil }
+
 // txJournal is a rotating log of transactions with the aim of storing locally
 // created transactions to allow non-executed ones to survive node restarts.
 type txJournal struct {
@@ -59,6 +68,10 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error {
 	}
 	defer input.Close()
 
+	// Temporarily discard any journal additions (don't double add on load)
+	journal.writer = new(devNull)
+	defer func() { journal.writer = nil }()
+
 	// Inject all transactions from the journal into the pool
 	stream := rlp.NewStream(input, 0)
 	total, dropped := 0, 0

core/tx_list.go

@@ -254,7 +254,10 @@ func (l *txList) Add(tx *types.Transaction, priceBump uint64) (bool, *types.Tran
 	old := l.txs.Get(tx.Nonce())
 	if old != nil {
 		threshold := new(big.Int).Div(new(big.Int).Mul(old.GasPrice(), big.NewInt(100+int64(priceBump))), big.NewInt(100))
-		if threshold.Cmp(tx.GasPrice()) >= 0 {
+		// Have to ensure that the new gas price is higher than the old gas
+		// price as well as checking the percentage threshold to ensure that
+		// this is accurate for low (Wei-level) gas price replacements
+		if old.GasPrice().Cmp(tx.GasPrice()) >= 0 || threshold.Cmp(tx.GasPrice()) > 0 {
 			return false, nil
 		}
 	}

core/tx_pool.go

@@ -103,6 +103,16 @@ var (
 	underpricedTxCounter = metrics.NewCounter("txpool/underpriced")
 )
 
+// TxStatus is the current status of a transaction as seen py the pool.
+type TxStatus uint
+
+const (
+	TxStatusUnknown TxStatus = iota
+	TxStatusQueued
+	TxStatusPending
+	TxStatusIncluded
+)
+
 // blockChain provides the state of blockchain and current gas limit to do
 // some pre checks in tx pool and event subscribers.
 type blockChain interface {
@@ -640,6 +650,10 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		pool.journalTx(from, tx)
 
 		log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To())
+
+		// We've directly injected a replacement transaction, notify subsystems
+		go pool.txFeed.Send(TxPreEvent{tx})
+
 		return old != nil, nil
 	}
 	// New transaction isn't replacing a pending one, push into queue
@@ -729,6 +743,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T
 	// Set the potentially new pending nonce and notify any subsystems of the new tx
 	pool.beats[addr] = time.Now()
 	pool.pendingState.SetNonce(addr, tx.Nonce()+1)
+
 	go pool.txFeed.Send(TxPreEvent{tx})
 }
 
@@ -749,14 +764,14 @@ func (pool *TxPool) AddRemote(tx *types.Transaction) error {
 // AddLocals enqueues a batch of transactions into the pool if they are valid,
 // marking the senders as a local ones in the mean time, ensuring they go around
 // the local pricing constraints.
-func (pool *TxPool) AddLocals(txs []*types.Transaction) error {
+func (pool *TxPool) AddLocals(txs []*types.Transaction) []error {
 	return pool.addTxs(txs, !pool.config.NoLocals)
 }
 
 // AddRemotes enqueues a batch of transactions into the pool if they are valid.
 // If the senders are not among the locally tracked ones, full pricing constraints
 // will apply.
-func (pool *TxPool) AddRemotes(txs []*types.Transaction) error {
+func (pool *TxPool) AddRemotes(txs []*types.Transaction) []error {
 	return pool.addTxs(txs, false)
 }
 
@@ -779,7 +794,7 @@ func (pool *TxPool) addTx(tx *types.Transaction, local bool) error {
 }
 
 // addTxs attempts to queue a batch of transactions if they are valid.
-func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error {
+func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 	pool.mu.Lock()
 	defer pool.mu.Unlock()
 
@@ -788,11 +803,14 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error {
 
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
-func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error {
+func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
 	// Add the batch of transaction, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
-	for _, tx := range txs {
-		if replace, err := pool.add(tx, local); err == nil {
+	errs := make([]error, len(txs))
+
+	for i, tx := range txs {
+		var replace bool
+		if replace, errs[i] = pool.add(tx, local); errs[i] == nil {
 			if !replace {
 				from, _ := types.Sender(pool.signer, tx) // already validated
 				dirty[from] = struct{}{}
@@ -802,12 +820,32 @@ func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error {
 	// Only reprocess the internal state if something was actually added
 	if len(dirty) > 0 {
 		addrs := make([]common.Address, 0, len(dirty))
-		for addr, _ := range dirty {
+		for addr := range dirty {
 			addrs = append(addrs, addr)
 		}
 		pool.promoteExecutables(addrs)
 	}
-	return nil
+	return errs
+}
+
+// Status returns the status (unknown/pending/queued) of a batch of transactions
+// identified by their hashes.
+func (pool *TxPool) Status(hashes []common.Hash) []TxStatus {
+	pool.mu.RLock()
+	defer pool.mu.RUnlock()
+
+	status := make([]TxStatus, len(hashes))
+	for i, hash := range hashes {
+		if tx := pool.all[hash]; tx != nil {
+			from, _ := types.Sender(pool.signer, tx) // already validated
+			if pool.pending[from].txs.items[tx.Nonce()] != nil {
+				status[i] = TxStatusPending
+			} else {
+				status[i] = TxStatusQueued
+			}
+		}
+	}
+	return status
 }
 
 // Get returns a transaction if it is contained in the pool
@@ -869,7 +907,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 	// Gather all the accounts potentially needing updates
 	if accounts == nil {
 		accounts = make([]common.Address, 0, len(pool.queue))
-		for addr, _ := range pool.queue {
+		for addr := range pool.queue {
 			accounts = append(accounts, addr)
 		}
 	}

core/tx_pool_test.go

@@ -105,7 +105,7 @@ func validateTxPoolInternals(pool *TxPool) error {
 	for addr, txs := range pool.pending {
 		// Find the last transaction
 		var last uint64
-		for nonce, _ := range txs.txs.items {
+		for nonce := range txs.txs.items {
 			if last < nonce {
 				last = nonce
 			}
@@ -117,6 +117,28 @@ func validateTxPoolInternals(pool *TxPool) error {
 	return nil
 }
 
+// validateEvents checks that the correct number of transaction addition events
+// were fired on the pool's event feed.
+func validateEvents(events chan TxPreEvent, count int) error {
+	for i := 0; i < count; i++ {
+		select {
+		case <-events:
+		case <-time.After(time.Second):
+			return fmt.Errorf("event #%d not fired", i)
+		}
+	}
+	select {
+	case tx := <-events:
+		return fmt.Errorf("more than %d events fired: %v", count, tx.Tx)
+
+	case <-time.After(50 * time.Millisecond):
+		// This branch should be "default", but it's a data race between goroutines,
+		// reading the event channel and pushng into it, so better wait a bit ensuring
+		// really nothing gets injected.
+	}
+	return nil
+}
+
 func deriveSender(tx *types.Transaction) (common.Address, error) {
 	return types.Sender(types.HomesteadSigner{}, tx)
 }
@@ -149,7 +171,9 @@ func (c *testChain) State() (*state.StateDB, error) {
 // This test simulates a scenario where a new block is imported during a
 // state reset and tests whether the pending state is in sync with the
 // block head event that initiated the resetState().
-func TestStateChangeDuringPoolReset(t *testing.T) {
+func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
+	t.Parallel()
+
 	var (
 		db, _      = ethdb.NewMemDatabase()
 		key, _     = crypto.GenerateKey()
@@ -201,6 +225,8 @@ func TestStateChangeDuringPoolReset(t *testing.T) {
 }
 
 func TestInvalidTransactions(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -236,6 +262,8 @@ func TestInvalidTransactions(t *testing.T) {
 }
 
 func TestTransactionQueue(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -287,7 +315,9 @@ func TestTransactionQueue(t *testing.T) {
 	}
 }
 
-func TestNegativeValue(t *testing.T) {
+func TestTransactionNegativeValue(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -300,6 +330,8 @@ func TestNegativeValue(t *testing.T) {
 }
 
 func TestTransactionChainFork(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -328,6 +360,8 @@ func TestTransactionChainFork(t *testing.T) {
 }
 
 func TestTransactionDoubleNonce(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -376,7 +410,9 @@ func TestTransactionDoubleNonce(t *testing.T) {
 	}
 }
 
-func TestMissingNonce(t *testing.T) {
+func TestTransactionMissingNonce(t *testing.T) {
+	t.Parallel()
+
 	pool, key := setupTxPool()
 	defer pool.Stop()
 
@@ -398,6 +434,8 @@ func TestMissingNonce(t *testing.T) {
 }
 
 func TestTransactionNonceRecovery(t *testing.T) {
+	t.Parallel()
+
 	const n = 10
 	pool, key := setupTxPool()
 	defer pool.Stop()
@@ -422,6 +460,8 @@ func TestTransactionNonceRecovery(t *testing.T) {
 // Tests that if an account runs out of funds, any pending and queued transactions
 // are dropped.
 func TestTransactionDropping(t *testing.T) {
+	t.Parallel()
+
 	// Create a test account and fund it
 	pool, key := setupTxPool()
 	defer pool.Stop()
@@ -515,6 +555,8 @@ func TestTransactionDropping(t *testing.T) {
 // of fund), all consecutive (still valid, but not executable) transactions are
 // postponed back into the future queue to prevent broadcasting them.
 func TestTransactionPostponing(t *testing.T) {
+	t.Parallel()
+
 	// Create a test account and fund it
 	pool, key := setupTxPool()
 	defer pool.Stop()
@@ -586,9 +628,68 @@ func TestTransactionPostponing(t *testing.T) {
 	}
 }
 
+// Tests that if the transaction pool has both executable and non-executable
+// transactions from an origin account, filling the nonce gap moves all queued
+// ones into the pending pool.
+func TestTransactionGapFilling(t *testing.T) {
+	t.Parallel()
+
+	// Create a test account and fund it
+	pool, key := setupTxPool()
+	defer pool.Stop()
+
+	account, _ := deriveSender(transaction(0, big.NewInt(0), key))
+	pool.currentState.AddBalance(account, big.NewInt(1000000))
+
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
+	// Create a pending and a queued transaction with a nonce-gap in between
+	if err := pool.AddRemote(transaction(0, big.NewInt(100000), key)); err != nil {
+		t.Fatalf("failed to add pending transaction: %v", err)
+	}
+	if err := pool.AddRemote(transaction(2, big.NewInt(100000), key)); err != nil {
+		t.Fatalf("failed to add queued transaction: %v", err)
+	}
+	pending, queued := pool.Stats()
+	if pending != 1 {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1)
+	}
+	if queued != 1 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1)
+	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+	// Fill the nonce gap and ensure all transactions become pending
+	if err := pool.AddRemote(transaction(1, big.NewInt(100000), key)); err != nil {
+		t.Fatalf("failed to add gapped transaction: %v", err)
+	}
+	pending, queued = pool.Stats()
+	if pending != 3 {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, 2); err != nil {
+		t.Fatalf("gap-filling event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+}
+
 // Tests that if the transaction count belonging to a single account goes above
 // some threshold, the higher transactions are dropped to prevent DOS attacks.
 func TestTransactionQueueAccountLimiting(t *testing.T) {
+	t.Parallel()
+
 	// Create a test account and fund it
 	pool, key := setupTxPool()
 	defer pool.Stop()
@@ -632,6 +733,8 @@ func TestTransactionQueueGlobalLimitingNoLocals(t *testing.T) {
 }
 
 func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) {
+	t.Parallel()
+
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -782,6 +885,8 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) {
 // above some threshold, as long as the transactions are executable, they are
 // accepted.
 func TestTransactionPendingLimiting(t *testing.T) {
+	t.Parallel()
+
 	// Create a test account and fund it
 	pool, key := setupTxPool()
 	defer pool.Stop()
@@ -789,6 +894,11 @@ func TestTransactionPendingLimiting(t *testing.T) {
 	account, _ := deriveSender(transaction(0, big.NewInt(0), key))
 	pool.currentState.AddBalance(account, big.NewInt(1000000))
 
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
 	// Keep queuing up transactions and make sure all above a limit are dropped
 	for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ {
 		if err := pool.AddRemote(transaction(i, big.NewInt(100000), key)); err != nil {
@@ -804,6 +914,12 @@ func TestTransactionPendingLimiting(t *testing.T) {
 	if len(pool.all) != int(testTxPoolConfig.AccountQueue+5) {
 		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), testTxPoolConfig.AccountQueue+5)
 	}
+	if err := validateEvents(events, int(testTxPoolConfig.AccountQueue+5)); err != nil {
+		t.Fatalf("event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
 }
 
 // Tests that the transaction limits are enforced the same way irrelevant whether
@@ -812,6 +928,8 @@ func TestTransactionQueueLimitingEquivalency(t *testing.T)   { testTransactionLi
 func TestTransactionPendingLimitingEquivalency(t *testing.T) { testTransactionLimitingEquivalency(t, 0) }
 
 func testTransactionLimitingEquivalency(t *testing.T, origin uint64) {
+	t.Parallel()
+
 	// Add a batch of transactions to a pool one by one
 	pool1, key1 := setupTxPool()
 	defer pool1.Stop()
@@ -859,6 +977,8 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) {
 // some hard threshold, the higher transactions are dropped to prevent DOS
 // attacks.
 func TestTransactionPendingGlobalLimiting(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -904,6 +1024,8 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) {
 
 // Tests that if transactions start being capped, transactions are also removed from 'all'
 func TestTransactionCapClearsFromAll(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -937,6 +1059,8 @@ func TestTransactionCapClearsFromAll(t *testing.T) {
 // some hard threshold, if they are under the minimum guaranteed slot count then
 // the transactions are still kept.
 func TestTransactionPendingMinimumAllowance(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -984,6 +1108,8 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) {
 //
 // Note, local transactions are never allowed to be dropped.
 func TestTransactionPoolRepricing(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -992,6 +1118,11 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
 	defer pool.Stop()
 
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
 	// Create a number of test accounts and fund them
 	keys := make([]*ecdsa.PrivateKey, 3)
 	for i := 0; i < len(keys); i++ {
@@ -1022,6 +1153,9 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if queued != 3 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3)
 	}
+	if err := validateEvents(events, 4); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1035,6 +1169,9 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if queued != 3 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3)
 	}
+	if err := validateEvents(events, 0); err != nil {
+		t.Fatalf("reprice event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1045,6 +1182,9 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced {
 		t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
+	if err := validateEvents(events, 0); err != nil {
+		t.Fatalf("post-reprice event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1056,6 +1196,9 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if pending, _ = pool.Stats(); pending != 3 {
 		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3)
 	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("post-reprice local event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1064,6 +1207,8 @@ func TestTransactionPoolRepricing(t *testing.T) {
 // Tests that setting the transaction pool gas price to a higher value does not
 // remove local transactions.
 func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -1125,6 +1270,8 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
 //
 // Note, local transactions are never allowed to be dropped.
 func TestTransactionPoolUnderpricing(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -1137,6 +1284,11 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	pool := NewTxPool(config, params.TestChainConfig, blockchain)
 	defer pool.Stop()
 
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
 	// Create a number of test accounts and fund them
 	keys := make([]*ecdsa.PrivateKey, 3)
 	for i := 0; i < len(keys); i++ {
@@ -1164,6 +1316,9 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 1 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1)
 	}
+	if err := validateEvents(events, 3); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1188,6 +1343,9 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 2 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
 	}
+	if err := validateEvents(events, 2); err != nil {
+		t.Fatalf("additional event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1203,6 +1361,9 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 2 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
 	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("local event firing failed: %v", err)
+	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
@@ -1211,6 +1372,8 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 // Tests that the pool rejects replacement transactions that don't meet the minimum
 // price bump required.
 func TestTransactionReplacement(t *testing.T) {
+	t.Parallel()
+
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
@@ -1219,6 +1382,11 @@ func TestTransactionReplacement(t *testing.T) {
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
 	defer pool.Stop()
 
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
 	// Create a test account to add transactions with
 	key, _ := crypto.GenerateKey()
 	pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000))
@@ -1236,35 +1404,45 @@ func TestTransactionReplacement(t *testing.T) {
 	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(2), key)); err != nil {
 		t.Fatalf("failed to replace original cheap pending transaction: %v", err)
 	}
+	if err := validateEvents(events, 2); err != nil {
+		t.Fatalf("cheap replacement event firing failed: %v", err)
+	}
 
 	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(price), key)); err != nil {
 		t.Fatalf("failed to add original proper pending transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced {
+	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100001), big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced {
 		t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced)
 	}
-	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil {
+	if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold), key)); err != nil {
 		t.Fatalf("failed to replace original proper pending transaction: %v", err)
 	}
+	if err := validateEvents(events, 2); err != nil {
+		t.Fatalf("proper replacement event firing failed: %v", err)
+	}
 	// Add queued transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too)
 	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), key)); err != nil {
-		t.Fatalf("failed to add original queued transaction: %v", err)
+		t.Fatalf("failed to add original cheap queued transaction: %v", err)
 	}
 	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced {
-		t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced)
+		t.Fatalf("original cheap queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced)
 	}
 	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(2), key)); err != nil {
-		t.Fatalf("failed to replace original queued transaction: %v", err)
+		t.Fatalf("failed to replace original cheap queued transaction: %v", err)
 	}
 
 	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(price), key)); err != nil {
-		t.Fatalf("failed to add original queued transaction: %v", err)
+		t.Fatalf("failed to add original proper queued transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100001), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced {
-		t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced)
+	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100001), big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced {
+		t.Fatalf("original proper queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil {
-		t.Fatalf("failed to replace original queued transaction: %v", err)
+	if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(threshold), key)); err != nil {
+		t.Fatalf("failed to replace original proper queued transaction: %v", err)
+	}
+
+	if err := validateEvents(events, 0); err != nil {
+		t.Fatalf("queued replacement event firing failed: %v", err)
 	}
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
@@ -1277,6 +1455,8 @@ func TestTransactionJournaling(t *testing.T)         { testTransactionJournaling
 func TestTransactionJournalingNoLocals(t *testing.T) { testTransactionJournaling(t, true) }
 
 func testTransactionJournaling(t *testing.T, nolocals bool) {
+	t.Parallel()
+
 	// Create a temporary file for the journal
 	file, err := ioutil.TempFile("", "")
 	if err != nil {
@@ -1335,6 +1515,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
 	pool.Stop()
 	statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1)
 	blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)}
+
 	pool = NewTxPool(config, params.TestChainConfig, blockchain)
 
 	pending, queued = pool.Stats()
@@ -1358,6 +1539,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
 	pool.lockedReset(nil, nil)
 	time.Sleep(2 * config.Rejournal)
 	pool.Stop()
+
 	statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1)
 	blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)}
 	pool = NewTxPool(config, params.TestChainConfig, blockchain)

core/types/transaction_test.go

@@ -147,12 +147,12 @@ func TestTransactionPriceNonceSort(t *testing.T) {
 	txset := NewTransactionsByPriceAndNonce(signer, groups)
 
 	txs := Transactions{}
-	for {
-		if tx := txset.Peek(); tx != nil {
-			txs = append(txs, tx)
-			txset.Shift()
-		}
-		break
+	for tx := txset.Peek(); tx != nil; tx = txset.Peek() {
+		txs = append(txs, tx)
+		txset.Shift()
+	}
+	if len(txs) != 25*25 {
+		t.Errorf("expected %d transactions, found %d", 25*25, len(txs))
 	}
 	for i, txi := range txs {
 		fromi, _ := Sender(signer, txi)

core/vm/analysis.go

@@ -25,7 +25,7 @@ import (
 // destinations stores one map per contract (keyed by hash of code).
 // The maps contain an entry for each location of a JUMPDEST
 // instruction.
-type destinations map[common.Hash][]byte
+type destinations map[common.Hash]bitvec
 
 // has checks whether code has a JUMPDEST at dest.
 func (d destinations) has(codehash common.Hash, code []byte, dest *big.Int) bool {
@@ -38,24 +38,53 @@ func (d destinations) has(codehash common.Hash, code []byte, dest *big.Int) bool
 
 	m, analysed := d[codehash]
 	if !analysed {
-		m = jumpdests(code)
+		m = codeBitmap(code)
 		d[codehash] = m
 	}
-	return (m[udest/8] & (1 << (udest % 8))) != 0
+	return OpCode(code[udest]) == JUMPDEST && m.codeSegment(udest)
 }
 
-// jumpdests creates a map that contains an entry for each
-// PC location that is a JUMPDEST instruction.
-func jumpdests(code []byte) []byte {
-	m := make([]byte, len(code)/8+1)
-	for pc := uint64(0); pc < uint64(len(code)); pc++ {
+// bitvec is a bit vector which maps bytes in a program.
+// An unset bit means the byte is an opcode, a set bit means
+// it's data (i.e. argument of PUSHxx).
+type bitvec []byte
+
+func (bits *bitvec) set(pos uint64) {
+	(*bits)[pos/8] |= 0x80 >> (pos % 8)
+}
+func (bits *bitvec) set8(pos uint64) {
+	(*bits)[pos/8] |= 0xFF >> (pos % 8)
+	(*bits)[pos/8+1] |= ^(0xFF >> (pos % 8))
+}
+
+// codeSegment checks if the position is in a code segment.
+func (bits *bitvec) codeSegment(pos uint64) bool {
+	return ((*bits)[pos/8] & (0x80 >> (pos % 8))) == 0
+}
+
+// codeBitmap collects data locations in code.
+func codeBitmap(code []byte) bitvec {
+	// The bitmap is 4 bytes longer than necessary, in case the code
+	// ends with a PUSH32, the algorithm will push zeroes onto the
+	// bitvector outside the bounds of the actual code.
+	bits := make(bitvec, len(code)/8+1+4)
+	for pc := uint64(0); pc < uint64(len(code)); {
 		op := OpCode(code[pc])
-		if op == JUMPDEST {
-			m[pc/8] |= 1 << (pc % 8)
-		} else if op >= PUSH1 && op <= PUSH32 {
-			a := uint64(op) - uint64(PUSH1) + 1
-			pc += a
+
+		if op >= PUSH1 && op <= PUSH32 {
+			numbits := op - PUSH1 + 1
+			pc++
+			for ; numbits >= 8; numbits -= 8 {
+				bits.set8(pc) // 8
+				pc += 8
+			}
+			for ; numbits > 0; numbits-- {
+				bits.set(pc)
+				pc++
+			}
+		} else {
+			pc++
 		}
 	}
-	return m
+	return bits
 }

core/vm/analysis_test.go

@@ -0,0 +1,53 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package vm
+
+import "testing"
+
+func TestJumpDestAnalysis(t *testing.T) {
+	tests := []struct {
+		code  []byte
+		exp   byte
+		which int
+	}{
+		{[]byte{byte(PUSH1), 0x01, 0x01, 0x01}, 0x40, 0},
+		{[]byte{byte(PUSH1), byte(PUSH1), byte(PUSH1), byte(PUSH1)}, 0x50, 0},
+		{[]byte{byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), 0x01, 0x01, 0x01}, 0x7F, 0},
+		{[]byte{byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x80, 1},
+		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), byte(PUSH2), byte(PUSH2), 0x01, 0x01, 0x01}, 0x03, 0},
+		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), 0x01, 0x01, 0x01, 0x01, 0x01}, 0x00, 1},
+		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x74, 0},
+		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x00, 1},
+		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x3F, 0},
+		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0xC0, 1},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x7F, 0},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0xFF, 1},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x80, 2},
+		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0x7f, 0},
+		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0xA0, 1},
+		{[]byte{byte(PUSH32)}, 0x7F, 0},
+		{[]byte{byte(PUSH32)}, 0xFF, 1},
+		{[]byte{byte(PUSH32)}, 0xFF, 2},
+	}
+	for _, test := range tests {
+		ret := codeBitmap(test.code)
+		if ret[test.which] != test.exp {
+			t.Fatalf("expected %x, got %02x", test.exp, ret[test.which])
+		}
+	}
+
+}

core/vm/interpreter.go

@@ -138,10 +138,10 @@ func (in *Interpreter) Run(snapshot int, contract *Contract, input []byte) (ret
 		pc   = uint64(0) // program counter
 		cost uint64
 		// copies used by tracer
-		stackCopy = newstack() // stackCopy needed for Tracer since stack is mutated by 63/64 gas rule 
-		pcCopy uint64 // needed for the deferred Tracer
-		gasCopy uint64 // for Tracer to log gas remaining before execution
-		logged bool // deferred Tracer should ignore already logged steps
+		stackCopy = newstack() // stackCopy needed for Tracer since stack is mutated by 63/64 gas rule
+		pcCopy    uint64       // needed for the deferred Tracer
+		gasCopy   uint64       // for Tracer to log gas remaining before execution
+		logged    bool         // deferred Tracer should ignore already logged steps
 	)
 	contract.Input = input
 
@@ -161,30 +161,27 @@ func (in *Interpreter) Run(snapshot int, contract *Contract, input []byte) (ret
 
 		if in.cfg.Debug {
 			logged = false
-			pcCopy = uint64(pc)
-			gasCopy = uint64(contract.Gas)
+			pcCopy = pc
+			gasCopy = contract.Gas
 			stackCopy = newstack()
 			for _, val := range stack.data {
 				stackCopy.push(val)
 			}
 		}
 
-		// get the operation from the jump table matching the opcode
+		// Get the operation from the jump table matching the opcode and validate the
+		// stack and make sure there enough stack items available to perform the operation
 		operation := in.cfg.JumpTable[op]
-		if err := in.enforceRestrictions(op, operation, stack); err != nil {
-			return nil, err
-		}
-
-		// if the op is invalid abort the process and return an error
 		if !operation.valid {
 			return nil, fmt.Errorf("invalid opcode 0x%x", int(op))
 		}
-
-		// validate the stack and make sure there enough stack items available
-		// to perform the operation
 		if err := operation.validateStack(stack); err != nil {
 			return nil, err
 		}
+		// If the operation is valid, enforce and write restrictions
+		if err := in.enforceRestrictions(op, operation, stack); err != nil {
+			return nil, err
+		}
 
 		var memorySize uint64
 		// calculate the new memory size and expand the memory to fit

core/vm/jump_table.go

@@ -42,12 +42,12 @@ type operation struct {
 	// memorySize returns the memory size required for the operation
 	memorySize memorySizeFunc
 
-	halts   bool // indicates whether the operation shoult halt further execution
+	halts   bool // indicates whether the operation should halt further execution
 	jumps   bool // indicates whether the program counter should not increment
 	writes  bool // determines whether this a state modifying operation
 	valid   bool // indication whether the retrieved operation is valid and known
 	reverts bool // determines whether the operation reverts state (implicitly halts)
-	returns bool // determines whether the opertions sets the return data content
+	returns bool // determines whether the operations sets the return data content
 }
 
 var (

core/vm/logger.go

@@ -45,7 +45,6 @@ type LogConfig struct {
 	DisableMemory  bool // disable memory capture
 	DisableStack   bool // disable stack capture
 	DisableStorage bool // disable storage capture
-	FullStorage    bool // show full storage (slow)
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
@@ -136,14 +135,13 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui
 		)
 		l.changedValues[contract.Address()][address] = value
 	}
-	// copy a snapstot of the current memory state to a new buffer
+	// Copy a snapstot of the current memory state to a new buffer
 	var mem []byte
 	if !l.cfg.DisableMemory {
 		mem = make([]byte, len(memory.Data()))
 		copy(mem, memory.Data())
 	}
-
-	// copy a snapshot of the current stack state to a new buffer
+	// Copy a snapshot of the current stack state to a new buffer
 	var stck []*big.Int
 	if !l.cfg.DisableStack {
 		stck = make([]*big.Int, len(stack.Data()))
@@ -151,26 +149,10 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui
 			stck[i] = new(big.Int).Set(item)
 		}
 	}
-
-	// Copy the storage based on the settings specified in the log config. If full storage
-	// is disabled (default) we can use the simple Storage.Copy method, otherwise we use
-	// the state object to query for all values (slow process).
+	// Copy a snapshot of the current storage to a new container
 	var storage Storage
 	if !l.cfg.DisableStorage {
-		if l.cfg.FullStorage {
-			storage = make(Storage)
-			// Get the contract account and loop over each storage entry. This may involve looping over
-			// the trie and is a very expensive process.
-
-			env.StateDB.ForEachStorage(contract.Address(), func(key, value common.Hash) bool {
-				storage[key] = value
-				// Return true, indicating we'd like to continue.
-				return true
-			})
-		} else {
-			// copy a snapshot of the current storage to a new container.
-			storage = l.changedValues[contract.Address()].Copy()
-		}
+		storage = l.changedValues[contract.Address()].Copy()
 	}
 	// create a new snaptshot of the EVM.
 	log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, err}

core/vm/logger_test.go

@@ -63,32 +63,8 @@ func TestStoreCapture(t *testing.T) {
 	if len(logger.changedValues[contract.Address()]) == 0 {
 		t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()]))
 	}
-
 	exp := common.BigToHash(big.NewInt(1))
 	if logger.changedValues[contract.Address()][index] != exp {
 		t.Errorf("expected %x, got %x", exp, logger.changedValues[contract.Address()][index])
 	}
 }
-
-func TestStorageCapture(t *testing.T) {
-	t.Skip("implementing this function is difficult. it requires all sort of interfaces to be implemented which isn't trivial. The value (the actual test) isn't worth it")
-	var (
-		ref      = &dummyContractRef{}
-		contract = NewContract(ref, ref, new(big.Int), 0)
-		env      = NewEVM(Context{}, dummyStateDB{ref: ref}, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
-		logger   = NewStructLogger(nil)
-		mem      = NewMemory()
-		stack    = newstack()
-	)
-
-	logger.CaptureState(env, 0, STOP, 0, 0, mem, stack, contract, 0, nil)
-	if ref.calledForEach {
-		t.Error("didn't expect for each to be called")
-	}
-
-	logger = NewStructLogger(&LogConfig{FullStorage: true})
-	logger.CaptureState(env, 0, STOP, 0, 0, mem, stack, contract, 0, nil)
-	if !ref.calledForEach {
-		t.Error("expected for each to be called")
-	}
-}

dashboard/README.md

@@ -0,0 +1,46 @@
+## Go Ethereum Dashboard
+
+The dashboard is a data visualizer integrated into geth, intended to collect and visualize useful information of an Ethereum node. It consists of two parts:
+
+* The client visualizes the collected data.
+* The server collects the data, and updates the clients.
+
+The client's UI uses [React][React] with JSX syntax, which is validated by the [ESLint][ESLint] linter mostly according to the [Airbnb React/JSX Style Guide][Airbnb]. The style is defined in the `.eslintrc` configuration file. The resources are bundled into a single `bundle.js` file using [Webpack][Webpack], which relies on the `webpack.config.js`. The bundled file is referenced from `dashboard.html` and takes part in the `assets.go` too. The necessary dependencies for the module bundler are gathered by [Node.js][Node.js].
+
+### Development and bundling
+
+As the dashboard depends on certain NPM packages (which are not included in the go-ethereum repo), these need to be installed first:
+
+```
+$ (cd dashboard/assets && npm install)
+```
+
+Normally the dashboard assets are bundled into Geth via `go-bindata` to avoid external dependencies. Rebuilding Geth after each UI modification however is not feasible from a developer perspective. Instead, we can run `webpack` in watch mode to automatically rebundle the UI, and ask `geth` to use external assets to not rely on compiled resources:
+
+```
+$ (cd dashboard/assets && ./node_modules/.bin/webpack --watch)
+$ geth --dashboard --dashboard.assets=dashboard/assets/public --vmodule=dashboard=5
+```
+
+To bundle up the final UI into Geth, run `webpack` and `go generate`:
+
+```
+$ (cd dashboard/assets && ./node_modules/.bin/webpack)
+$ go generate ./dashboard
+```
+
+### Have fun
+
+[Webpack][Webpack] offers handy tools for visualizing the bundle's dependency tree and space usage.
+
+* Generate the bundle's profile running `webpack --profile --json > stats.json`
+* For the _dependency tree_ go to [Webpack Analyze][WA], and import `stats.json`
+* For the _space usage_ go to [Webpack Visualizer][WV], and import `stats.json`
+
+[React]: https://reactjs.org/
+[ESLint]: https://eslint.org/
+[Airbnb]: https://github.com/airbnb/javascript/tree/master/react
+[Webpack]: https://webpack.github.io/
+[WA]: http://webpack.github.io/analyse/
+[WV]: http://chrisbateman.github.io/webpack-visualizer/
+[Node.js]: https://nodejs.org/en/

dashboard/assets/.eslintrc

@@ -0,0 +1,52 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// React syntax style mostly according to https://github.com/airbnb/javascript/tree/master/react
+{
+  "plugins": [
+    "react"
+  ],
+  "parser": "babel-eslint",
+  "parserOptions": {
+    "ecmaFeatures": {
+      "jsx": true,
+      "modules": true
+    }
+  },
+  "rules": {
+    "react/prefer-es6-class": 2,
+    "react/prefer-stateless-function": 2,
+    "react/jsx-pascal-case": 2,
+    "react/jsx-closing-bracket-location": [1, {"selfClosing": "tag-aligned", "nonEmpty": "tag-aligned"}],
+    "react/jsx-closing-tag-location": 1,
+    "jsx-quotes": ["error", "prefer-double"],
+    "no-multi-spaces": "error",
+    "react/jsx-tag-spacing": 2,
+    "react/jsx-curly-spacing": [2, {"when": "never", "children": true}],
+    "react/jsx-boolean-value": 2,
+    "react/no-string-refs": 2,
+    "react/jsx-wrap-multilines": 2,
+    "react/self-closing-comp": 2,
+    "react/jsx-no-bind": 2,
+    "react/require-render-return": 2,
+    "react/no-is-mounted": 2,
+    "key-spacing": ["error", {"align": {
+      "beforeColon": false,
+      "afterColon": true,
+      "on": "value"
+    }}]
+  }
+}

dashboard/assets/components/Common.jsx

@@ -0,0 +1,52 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// isNullOrUndefined returns true if the given variable is null or undefined.
+export const isNullOrUndefined = variable => variable === null || typeof variable === 'undefined';
+
+export const LIMIT = {
+    memory:  200, // Maximum number of memory data samples.
+    traffic: 200, // Maximum number of traffic data samples.
+    log:     200, // Maximum number of logs.
+};
+// The sidebar menu and the main content are rendered based on these elements.
+export const TAGS = (() => {
+    const T = {
+        home:         { title: "Home", },
+        chain:        { title: "Chain", },
+        transactions: { title: "Transactions", },
+        network:      { title: "Network", },
+        system:       { title: "System", },
+        logs:         { title: "Logs", },
+    };
+    // Using the key is circumstantial in some cases, so it is better to insert it also as a value.
+    // This way the mistyping is prevented.
+    for(let key in T) {
+        T[key]['id'] = key;
+    }
+    return T;
+})();
+
+export const DATA_KEYS = (() => {
+    const DK = {};
+    ["memory", "traffic", "logs"].map(key => {
+       DK[key] = key;
+    });
+    return DK;
+})();
+
+// Temporary - taken from Material-UI
+export const DRAWER_WIDTH = 240;

dashboard/assets/components/Dashboard.jsx

@@ -0,0 +1,169 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import {withStyles} from 'material-ui/styles';
+
+import SideBar from './SideBar.jsx';
+import Header from './Header.jsx';
+import Main from "./Main.jsx";
+import {isNullOrUndefined, LIMIT, TAGS, DATA_KEYS,} from "./Common.jsx";
+
+// Styles for the Dashboard component.
+const styles = theme => ({
+    appFrame: {
+        position:   'relative',
+        display:    'flex',
+        width:      '100%',
+        height:     '100%',
+        background: theme.palette.background.default,
+    },
+});
+
+// Dashboard is the main component, which renders the whole page, makes connection with the server and listens for messages.
+// When there is an incoming message, updates the page's content correspondingly.
+class Dashboard extends Component {
+    constructor(props) {
+        super(props);
+        this.state = {
+            active:       TAGS.home.id, // active menu
+            sideBar:      true, // true if the sidebar is opened
+            memory:       [],
+            traffic:      [],
+            logs:         [],
+            shouldUpdate: {},
+        };
+    }
+
+    // componentDidMount initiates the establishment of the first websocket connection after the component is rendered.
+    componentDidMount() {
+        this.reconnect();
+    }
+
+    // reconnect establishes a websocket connection with the server, listens for incoming messages
+    // and tries to reconnect on connection loss.
+    reconnect = () => {
+        const server = new WebSocket(((window.location.protocol === "https:") ? "wss://" : "ws://") + window.location.host + "/api");
+
+        server.onmessage = event => {
+            const msg = JSON.parse(event.data);
+            if (isNullOrUndefined(msg)) {
+                return;
+            }
+            this.update(msg);
+        };
+
+        server.onclose = () => {
+            setTimeout(this.reconnect, 3000);
+        };
+    };
+
+    // update analyzes the incoming message, and updates the charts' content correspondingly.
+    update = msg => {
+        console.log(msg);
+        this.setState(prevState => {
+            let newState = [];
+            newState.shouldUpdate = {};
+            const insert = (key, values, limit) => {
+                newState[key] = [...prevState[key], ...values];
+                while (newState[key].length > limit) {
+                    newState[key].shift();
+                }
+                newState.shouldUpdate[key] = true;
+            };
+            // (Re)initialize the state with the past data.
+            if (!isNullOrUndefined(msg.history)) {
+                const memory = DATA_KEYS.memory;
+                const traffic = DATA_KEYS.traffic;
+                newState[memory] = [];
+                newState[traffic] = [];
+                if (!isNullOrUndefined(msg.history.memorySamples)) {
+                    newState[memory] = msg.history.memorySamples.map(elem => isNullOrUndefined(elem.value) ? 0 : elem.value);
+                    while (newState[memory].length > LIMIT.memory) {
+                        newState[memory].shift();
+                    }
+                    newState.shouldUpdate[memory] = true;
+                }
+                if (!isNullOrUndefined(msg.history.trafficSamples)) {
+                    newState[traffic] = msg.history.trafficSamples.map(elem => isNullOrUndefined(elem.value) ? 0 : elem.value);
+                    while (newState[traffic].length > LIMIT.traffic) {
+                        newState[traffic].shift();
+                    }
+                    newState.shouldUpdate[traffic] = true;
+                }
+            }
+            // Insert the new data samples.
+            if (!isNullOrUndefined(msg.memory)) {
+                insert(DATA_KEYS.memory, [isNullOrUndefined(msg.memory.value) ? 0 : msg.memory.value], LIMIT.memory);
+            }
+            if (!isNullOrUndefined(msg.traffic)) {
+                insert(DATA_KEYS.traffic, [isNullOrUndefined(msg.traffic.value) ? 0 : msg.traffic.value], LIMIT.traffic);
+            }
+            if (!isNullOrUndefined(msg.log)) {
+                insert(DATA_KEYS.logs, [msg.log], LIMIT.log);
+            }
+
+            return newState;
+        });
+    };
+
+    // The change of the active label on the SideBar component will trigger a new render in the Main component.
+    changeContent = active => {
+        this.setState(prevState => prevState.active !== active ? {active: active} : {});
+    };
+
+    openSideBar = () => {
+        this.setState({sideBar: true});
+    };
+
+    closeSideBar = () => {
+        this.setState({sideBar: false});
+    };
+
+    render() {
+        // The classes property is injected by withStyles().
+        const {classes} = this.props;
+
+        return (
+            <div className={classes.appFrame}>
+                <Header
+                    opened={this.state.sideBar}
+                    open={this.openSideBar}
+                />
+                <SideBar
+                    opened={this.state.sideBar}
+                    close={this.closeSideBar}
+                    changeContent={this.changeContent}
+                />
+                <Main
+                    opened={this.state.sideBar}
+                    active={this.state.active}
+                    memory={this.state.memory}
+                    traffic={this.state.traffic}
+                    logs={this.state.logs}
+                    shouldUpdate={this.state.shouldUpdate}
+                />
+            </div>
+        );
+    }
+}
+
+Dashboard.propTypes = {
+    classes: PropTypes.object.isRequired,
+};
+
+export default withStyles(styles)(Dashboard);

dashboard/assets/components/Header.jsx

@@ -0,0 +1,87 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import classNames from 'classnames';
+import {withStyles} from 'material-ui/styles';
+import AppBar from 'material-ui/AppBar';
+import Toolbar from 'material-ui/Toolbar';
+import Typography from 'material-ui/Typography';
+import IconButton from 'material-ui/IconButton';
+import MenuIcon from 'material-ui-icons/Menu';
+
+import {DRAWER_WIDTH} from './Common.jsx';
+
+// Styles for the Header component.
+const styles = theme => ({
+    appBar: {
+        position:   'absolute',
+        transition: theme.transitions.create(['margin', 'width'], {
+            easing:   theme.transitions.easing.sharp,
+            duration: theme.transitions.duration.leavingScreen,
+        }),
+    },
+    appBarShift: {
+        marginLeft: DRAWER_WIDTH,
+        width:      `calc(100% - ${DRAWER_WIDTH}px)`,
+        transition: theme.transitions.create(['margin', 'width'], {
+            easing:   theme.transitions.easing.easeOut,
+            duration: theme.transitions.duration.enteringScreen,
+        }),
+    },
+    menuButton: {
+        marginLeft:  12,
+        marginRight: 20,
+    },
+    hide: {
+        display: 'none',
+    },
+});
+
+// Header renders a header, which contains a sidebar opener icon when that is closed.
+class Header extends Component {
+    render() {
+        // The classes property is injected by withStyles().
+        const {classes} = this.props;
+
+        return (
+            <AppBar className={classNames(classes.appBar, this.props.opened && classes.appBarShift)}>
+                <Toolbar disableGutters={!this.props.opened}>
+                    <IconButton
+                        color="contrast"
+                        aria-label="open drawer"
+                        onClick={this.props.open}
+                        className={classNames(classes.menuButton, this.props.opened && classes.hide)}
+                    >
+                        <MenuIcon />
+                    </IconButton>
+                    <Typography type="title" color="inherit" noWrap>
+                        Go Ethereum Dashboard
+                    </Typography>
+                </Toolbar>
+            </AppBar>
+        );
+    }
+}
+
+Header.propTypes = {
+    classes: PropTypes.object.isRequired,
+    opened:  PropTypes.bool.isRequired,
+    open:    PropTypes.func.isRequired,
+};
+
+export default withStyles(styles)(Header);

dashboard/assets/components/Home.jsx

@@ -0,0 +1,89 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import Grid from 'material-ui/Grid';
+import {LineChart, AreaChart, Area, YAxis, CartesianGrid, Line, ResponsiveContainer} from 'recharts';
+import {withTheme} from 'material-ui/styles';
+
+import {isNullOrUndefined, DATA_KEYS} from "./Common.jsx";
+
+// ChartGrid renders a grid container for responsive charts.
+// The children are Recharts components extended with the Material-UI's xs property.
+class ChartGrid extends Component {
+    render() {
+        return (
+            <Grid container spacing={this.props.spacing}>
+                {
+                    React.Children.map(this.props.children, child => (
+                        <Grid item xs={child.props.xs}>
+                            <ResponsiveContainer width="100%" height={child.props.height}>
+                                {React.cloneElement(child, {data: child.props.values.map(value => ({value: value}))})}
+                            </ResponsiveContainer>
+                        </Grid>
+                    ))
+                }
+            </Grid>
+        );
+    }
+}
+
+ChartGrid.propTypes = {
+    spacing: PropTypes.number.isRequired,
+};
+
+// Home renders the home component.
+class Home extends Component {
+    shouldComponentUpdate(nextProps) {
+        return !isNullOrUndefined(nextProps.shouldUpdate[DATA_KEYS.memory]) ||
+            !isNullOrUndefined(nextProps.shouldUpdate[DATA_KEYS.traffic]);
+    }
+
+    render() {
+        const {theme} = this.props;
+        const memoryColor = theme.palette.primary[300];
+        const trafficColor = theme.palette.secondary[300];
+
+        return (
+            <ChartGrid spacing={24}>
+                <AreaChart xs={6} height={300} values={this.props.memory}>
+                    <YAxis />
+                    <Area type="monotone" dataKey="value" stroke={memoryColor} fill={memoryColor} />
+                </AreaChart>
+                <LineChart xs={6} height={300} values={this.props.traffic}>
+                    <Line type="monotone" dataKey="value" stroke={trafficColor} dot={false} />
+                </LineChart>
+                <LineChart xs={6} height={300} values={this.props.memory}>
+                    <YAxis />
+                    <CartesianGrid stroke="#eee" strokeDasharray="5 5" />
+                    <Line type="monotone" dataKey="value" stroke={memoryColor} dot={false} />
+                </LineChart>
+                <AreaChart xs={6} height={300} values={this.props.traffic}>
+                    <CartesianGrid stroke="#eee" strokeDasharray="5 5" vertical={false} />
+                    <Area type="monotone" dataKey="value" stroke={trafficColor} fill={trafficColor} />
+                </AreaChart>
+            </ChartGrid>
+        );
+    }
+}
+
+Home.propTypes = {
+    theme:        PropTypes.object.isRequired,
+    shouldUpdate: PropTypes.object.isRequired,
+};
+
+export default withTheme()(Home);

dashboard/assets/components/Main.jsx

@@ -0,0 +1,109 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import classNames from 'classnames';
+import {withStyles} from 'material-ui/styles';
+
+import {TAGS, DRAWER_WIDTH} from "./Common.jsx";
+import Home from './Home.jsx';
+
+// ContentSwitch chooses and renders the proper page content.
+class ContentSwitch extends Component {
+    render() {
+        switch(this.props.active) {
+            case TAGS.home.id:
+                return <Home memory={this.props.memory} traffic={this.props.traffic} shouldUpdate={this.props.shouldUpdate} />;
+            case TAGS.chain.id:
+                return null;
+            case TAGS.transactions.id:
+                return null;
+            case TAGS.network.id:
+                // Only for testing.
+                return null;
+            case TAGS.system.id:
+                return null;
+            case TAGS.logs.id:
+                return <div>{this.props.logs.map((log, index) => <div key={index}>{log}</div>)}</div>;
+        }
+        return null;
+    }
+}
+
+ContentSwitch.propTypes = {
+    active:       PropTypes.string.isRequired,
+    shouldUpdate: PropTypes.object.isRequired,
+};
+
+// styles contains the styles for the Main component.
+const styles = theme => ({
+    content: {
+        width:           '100%',
+        marginLeft:      -DRAWER_WIDTH,
+        flexGrow:        1,
+        backgroundColor: theme.palette.background.default,
+        padding:         theme.spacing.unit * 3,
+        transition:      theme.transitions.create('margin', {
+            easing:   theme.transitions.easing.sharp,
+            duration: theme.transitions.duration.leavingScreen,
+        }),
+        marginTop:                    56,
+        overflow:                     'auto',
+        [theme.breakpoints.up('sm')]: {
+            content: {
+                height:    'calc(100% - 64px)',
+                marginTop: 64,
+            },
+        },
+    },
+    contentShift: {
+        marginLeft: 0,
+        transition: theme.transitions.create('margin', {
+            easing:   theme.transitions.easing.easeOut,
+            duration: theme.transitions.duration.enteringScreen,
+        }),
+    },
+});
+
+// Main renders a component for the page content.
+class Main extends Component {
+    render() {
+        // The classes property is injected by withStyles().
+        const {classes} = this.props;
+
+        return (
+            <main className={classNames(classes.content, this.props.opened && classes.contentShift)}>
+                <ContentSwitch
+                    active={this.props.active}
+                    memory={this.props.memory}
+                    traffic={this.props.traffic}
+                    logs={this.props.logs}
+                    shouldUpdate={this.props.shouldUpdate}
+                />
+            </main>
+        );
+    }
+}
+
+Main.propTypes = {
+    classes:      PropTypes.object.isRequired,
+    opened:       PropTypes.bool.isRequired,
+    active:       PropTypes.string.isRequired,
+    shouldUpdate: PropTypes.object.isRequired,
+};
+
+export default withStyles(styles)(Main);

dashboard/assets/components/SideBar.jsx

@@ -0,0 +1,106 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import {withStyles} from 'material-ui/styles';
+import Drawer from 'material-ui/Drawer';
+import {IconButton} from "material-ui";
+import List, {ListItem, ListItemText} from 'material-ui/List';
+import ChevronLeftIcon from 'material-ui-icons/ChevronLeft';
+
+import {TAGS, DRAWER_WIDTH} from './Common.jsx';
+
+// Styles for the SideBar component.
+const styles = theme => ({
+    drawerPaper: {
+        position: 'relative',
+        height:   '100%',
+        width:    DRAWER_WIDTH,
+    },
+    drawerHeader: {
+        display:            'flex',
+        alignItems:         'center',
+        justifyContent:     'flex-end',
+        padding:            '0 8px',
+        ...theme.mixins.toolbar,
+        transitionDuration: {
+            enter: theme.transitions.duration.enteringScreen,
+            exit:  theme.transitions.duration.leavingScreen,
+        }
+    },
+});
+
+// SideBar renders a sidebar component.
+class SideBar extends Component {
+    constructor(props) {
+        super(props);
+
+        // clickOn contains onClick event functions for the menu items.
+        // Instantiate only once, and reuse the existing functions to prevent the creation of
+        // new function instances every time the render method is triggered.
+        this.clickOn = {};
+        for(let key in TAGS) {
+            const id = TAGS[key].id;
+            this.clickOn[id] = event => {
+                event.preventDefault();
+                console.log(event.target.key);
+                this.props.changeContent(id);
+            };
+        }
+    }
+
+    render() {
+        // The classes property is injected by withStyles().
+        const {classes} = this.props;
+
+        return (
+            <Drawer
+                type="persistent"
+                classes={{paper: classes.drawerPaper,}}
+                open={this.props.opened}
+            >
+                <div>
+                    <div className={classes.drawerHeader}>
+                        <IconButton onClick={this.props.close}>
+                            <ChevronLeftIcon />
+                        </IconButton>
+                    </div>
+                    <List>
+                        {
+                            Object.values(TAGS).map(tag => {
+                                return (
+                                    <ListItem button key={tag.id} onClick={this.clickOn[tag.id]}>
+                                        <ListItemText primary={tag.title} />
+                                    </ListItem>
+                                );
+                            })
+                        }
+                    </List>
+                </div>
+            </Drawer>
+        );
+    }
+}
+
+SideBar.propTypes = {
+    classes:       PropTypes.object.isRequired,
+    opened:        PropTypes.bool.isRequired,
+    close:         PropTypes.func.isRequired,
+    changeContent: PropTypes.func.isRequired,
+};
+
+export default withStyles(styles)(SideBar);

dashboard/assets/index.jsx

@@ -0,0 +1,36 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React from 'react';
+import {hydrate} from 'react-dom';
+import {createMuiTheme, MuiThemeProvider} from 'material-ui/styles';
+
+import Dashboard from './components/Dashboard.jsx';
+
+// Theme for the dashboard.
+const theme = createMuiTheme({
+    palette: {
+        type: 'dark',
+    },
+});
+
+// Renders the whole dashboard.
+hydrate(
+    <MuiThemeProvider theme={theme}>
+        <Dashboard />
+    </MuiThemeProvider>,
+    document.getElementById('dashboard')
+);

dashboard/assets/package.json

@@ -0,0 +1,22 @@
+{
+  "dependencies": {
+    "babel-core": "^6.26.0",
+    "babel-eslint": "^8.0.1",
+    "babel-loader": "^7.1.2",
+    "babel-preset-env": "^1.6.1",
+    "babel-preset-react": "^6.24.1",
+    "babel-preset-stage-0": "^6.24.1",
+    "classnames": "^2.2.5",
+    "eslint": "^4.5.0",
+    "eslint-plugin-react": "^7.4.0",
+    "material-ui": "^1.0.0-beta.18",
+    "material-ui-icons": "^1.0.0-beta.17",
+    "path": "^0.12.7",
+    "prop-types": "^15.6.0",
+    "recharts": "^1.0.0-beta.0",
+    "react": "^16.0.0",
+    "react-dom": "^16.0.0",
+    "url": "^0.11.0",
+    "webpack": "^3.5.5"
+  }
+}

dashboard/assets/public/dashboard.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en" style="height: 100%">
+    <head>
+        <meta charset="UTF-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+
+        <title>Go Ethereum Dashboard</title>
+        <link rel="shortcut icon" type="image/ico" href="https://ethereum.org/favicon.ico"/>
+
+        <!-- TODO (kurkomisi): Return to the external libraries to speed up the bundling during development -->
+    </head>
+    <body style="height: 100%; margin: 0">
+        <div id="dashboard" style="height: 100%"></div>
+        <script src="bundle.js"></script>
+    </body>
+</html>

dashboard/assets/webpack.config.js

@@ -0,0 +1,36 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+const path = require('path');
+
+module.exports = {
+    entry:  './index.jsx',
+    output: {
+        path:     path.resolve(__dirname, 'public'),
+        filename: 'bundle.js',
+    },
+    module: {
+        loaders: [
+            {
+                test: /\.jsx$/, // regexp for JSX files
+                loader: 'babel-loader', // The babel configuration is in the package.json.
+                query: {
+                    presets: ['env', 'react', 'stage-0']
+                }
+            },
+        ],
+    },
+};

dashboard/config.go

@@ -0,0 +1,45 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package dashboard
+
+import "time"
+
+// DefaultConfig contains default settings for the dashboard.
+var DefaultConfig = Config{
+	Host:    "localhost",
+	Port:    8080,
+	Refresh: 3 * time.Second,
+}
+
+// Config contains the configuration parameters of the dashboard.
+type Config struct {
+	// Host is the host interface on which to start the dashboard server. If this
+	// field is empty, no dashboard will be started.
+	Host string `toml:",omitempty"`
+
+	// Port is the TCP port number on which to start the dashboard server. The
+	// default zero value is/ valid and will pick a port number randomly (useful
+	// for ephemeral nodes).
+	Port int `toml:",omitempty"`
+
+	// Refresh is the refresh rate of the data updates, the chartEntry will be collected this often.
+	Refresh time.Duration `toml:",omitempty"`
+
+	// Assets offers a possibility to manually set the dashboard website's location on the server side.
+	// It is useful for debugging, avoids the repeated generation of the binary.
+	Assets string `toml:",omitempty"`
+}

dashboard/dashboard.go

@@ -0,0 +1,305 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package dashboard
+
+//go:generate go-bindata -nometadata -o assets.go -prefix assets -pkg dashboard assets/public/...
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/rcrowley/go-metrics"
+	"golang.org/x/net/websocket"
+)
+
+const (
+	memorySampleLimit  = 200 // Maximum number of memory data samples
+	trafficSampleLimit = 200 // Maximum number of traffic data samples
+)
+
+var nextId uint32 // Next connection id
+
+// Dashboard contains the dashboard internals.
+type Dashboard struct {
+	config *Config
+
+	listener net.Listener
+	conns    map[uint32]*client // Currently live websocket connections
+	charts   charts             // The collected data samples to plot
+	lock     sync.RWMutex       // Lock protecting the dashboard's internals
+
+	quit chan chan error // Channel used for graceful exit
+	wg   sync.WaitGroup
+}
+
+// message embraces the data samples of a client message.
+type message struct {
+	History *charts     `json:"history,omitempty"` // Past data samples
+	Memory  *chartEntry `json:"memory,omitempty"`  // One memory sample
+	Traffic *chartEntry `json:"traffic,omitempty"` // One traffic sample
+	Log     string      `json:"log,omitempty"`     // One log
+}
+
+// client represents active websocket connection with a remote browser.
+type client struct {
+	conn   *websocket.Conn // Particular live websocket connection
+	msg    chan message    // Message queue for the update messages
+	logger log.Logger      // Logger for the particular live websocket connection
+}
+
+// charts contains the collected data samples.
+type charts struct {
+	Memory  []*chartEntry `json:"memorySamples,omitempty"`
+	Traffic []*chartEntry `json:"trafficSamples,omitempty"`
+}
+
+// chartEntry represents one data sample
+type chartEntry struct {
+	Time  time.Time `json:"time,omitempty"`
+	Value float64   `json:"value,omitempty"`
+}
+
+// New creates a new dashboard instance with the given configuration.
+func New(config *Config) (*Dashboard, error) {
+	return &Dashboard{
+		conns:  make(map[uint32]*client),
+		config: config,
+		quit:   make(chan chan error),
+	}, nil
+}
+
+// Protocols is a meaningless implementation of node.Service.
+func (db *Dashboard) Protocols() []p2p.Protocol { return nil }
+
+// APIs is a meaningless implementation of node.Service.
+func (db *Dashboard) APIs() []rpc.API { return nil }
+
+// Start implements node.Service, starting the data collection thread and the listening server of the dashboard.
+func (db *Dashboard) Start(server *p2p.Server) error {
+	db.wg.Add(2)
+	go db.collectData()
+	go db.collectLogs() // In case of removing this line change 2 back to 1 in wg.Add.
+
+	http.HandleFunc("/", db.webHandler)
+	http.Handle("/api", websocket.Handler(db.apiHandler))
+
+	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", db.config.Host, db.config.Port))
+	if err != nil {
+		return err
+	}
+	db.listener = listener
+
+	go http.Serve(listener, nil)
+
+	return nil
+}
+
+// Stop implements node.Service, stopping the data collection thread and the connection listener of the dashboard.
+func (db *Dashboard) Stop() error {
+	// Close the connection listener.
+	var errs []error
+	if err := db.listener.Close(); err != nil {
+		errs = append(errs, err)
+	}
+	// Close the collectors.
+	errc := make(chan error, 1)
+	for i := 0; i < 2; i++ {
+		db.quit <- errc
+		if err := <-errc; err != nil {
+			errs = append(errs, err)
+		}
+	}
+	// Close the connections.
+	db.lock.Lock()
+	for _, c := range db.conns {
+		if err := c.conn.Close(); err != nil {
+			c.logger.Warn("Failed to close connection", "err", err)
+		}
+	}
+	db.lock.Unlock()
+
+	// Wait until every goroutine terminates.
+	db.wg.Wait()
+	log.Info("Dashboard stopped")
+
+	var err error
+	if len(errs) > 0 {
+		err = fmt.Errorf("%v", errs)
+	}
+
+	return err
+}
+
+// webHandler handles all non-api requests, simply flattening and returning the dashboard website.
+func (db *Dashboard) webHandler(w http.ResponseWriter, r *http.Request) {
+	log.Debug("Request", "URL", r.URL)
+
+	path := r.URL.String()
+	if path == "/" {
+		path = "/dashboard.html"
+	}
+	// If the path of the assets is manually set
+	if db.config.Assets != "" {
+		blob, err := ioutil.ReadFile(filepath.Join(db.config.Assets, path))
+		if err != nil {
+			log.Warn("Failed to read file", "path", path, "err", err)
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		w.Write(blob)
+		return
+	}
+	blob, err := Asset(filepath.Join("public", path))
+	if err != nil {
+		log.Warn("Failed to load the asset", "path", path, "err", err)
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	w.Write(blob)
+}
+
+// apiHandler handles requests for the dashboard.
+func (db *Dashboard) apiHandler(conn *websocket.Conn) {
+	id := atomic.AddUint32(&nextId, 1)
+	client := &client{
+		conn:   conn,
+		msg:    make(chan message, 128),
+		logger: log.New("id", id),
+	}
+	done := make(chan struct{}) // Buffered channel as sender may exit early
+
+	// Start listening for messages to send.
+	db.wg.Add(1)
+	go func() {
+		defer db.wg.Done()
+
+		for {
+			select {
+			case <-done:
+				return
+			case msg := <-client.msg:
+				if err := websocket.JSON.Send(client.conn, msg); err != nil {
+					client.logger.Warn("Failed to send the message", "msg", msg, "err", err)
+					client.conn.Close()
+					return
+				}
+			}
+		}
+	}()
+	// Send the past data.
+	client.msg <- message{
+		History: &db.charts,
+	}
+	// Start tracking the connection and drop at connection loss.
+	db.lock.Lock()
+	db.conns[id] = client
+	db.lock.Unlock()
+	defer func() {
+		db.lock.Lock()
+		delete(db.conns, id)
+		db.lock.Unlock()
+	}()
+	for {
+		fail := []byte{}
+		if _, err := conn.Read(fail); err != nil {
+			close(done)
+			return
+		}
+		// Ignore all messages
+	}
+}
+
+// collectData collects the required data to plot on the dashboard.
+func (db *Dashboard) collectData() {
+	defer db.wg.Done()
+
+	for {
+		select {
+		case errc := <-db.quit:
+			errc <- nil
+			return
+		case <-time.After(db.config.Refresh):
+			inboundTraffic := metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Rate1()
+			memoryInUse := metrics.DefaultRegistry.Get("system/memory/inuse").(metrics.Meter).Rate1()
+			now := time.Now()
+			memory := &chartEntry{
+				Time:  now,
+				Value: memoryInUse,
+			}
+			traffic := &chartEntry{
+				Time:  now,
+				Value: inboundTraffic,
+			}
+			// Remove the first elements in case the samples' amount exceeds the limit.
+			first := 0
+			if len(db.charts.Memory) == memorySampleLimit {
+				first = 1
+			}
+			db.charts.Memory = append(db.charts.Memory[first:], memory)
+			first = 0
+			if len(db.charts.Traffic) == trafficSampleLimit {
+				first = 1
+			}
+			db.charts.Traffic = append(db.charts.Traffic[first:], traffic)
+
+			db.sendToAll(&message{
+				Memory:  memory,
+				Traffic: traffic,
+			})
+		}
+	}
+}
+
+// collectLogs collects and sends the logs to the active dashboards.
+func (db *Dashboard) collectLogs() {
+	defer db.wg.Done()
+
+	// TODO (kurkomisi): log collection comes here.
+	for {
+		select {
+		case errc := <-db.quit:
+			errc <- nil
+			return
+		case <-time.After(db.config.Refresh / 2):
+			db.sendToAll(&message{
+				Log: "This is a fake log.",
+			})
+		}
+	}
+}
+
+// sendToAll sends the given message to the active dashboards.
+func (db *Dashboard) sendToAll(msg *message) {
+	db.lock.Lock()
+	for _, c := range db.conns {
+		select {
+		case c.msg <- *msg:
+		default:
+			c.conn.Close()
+		}
+	}
+	db.lock.Unlock()
+}

eth/api.go

@@ -636,3 +636,86 @@ func storageRangeAt(st state.Trie, start []byte, maxResult int) StorageRangeResu
 	}
 	return result
 }
+
+// GetModifiedAccountsByumber returns all accounts that have changed between the
+// two blocks specified. A change is defined as a difference in nonce, balance,
+// code hash, or storage hash.
+//
+// With one parameter, returns the list of accounts modified in the specified block.
+func (api *PrivateDebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) {
+	var startBlock, endBlock *types.Block
+
+	startBlock = api.eth.blockchain.GetBlockByNumber(startNum)
+	if startBlock == nil {
+		return nil, fmt.Errorf("start block %x not found", startNum)
+	}
+
+	if endNum == nil {
+		endBlock = startBlock
+		startBlock = api.eth.blockchain.GetBlockByHash(startBlock.ParentHash())
+		if startBlock == nil {
+			return nil, fmt.Errorf("block %x has no parent", endBlock.Number())
+		}
+	} else {
+		endBlock = api.eth.blockchain.GetBlockByNumber(*endNum)
+		if endBlock == nil {
+			return nil, fmt.Errorf("end block %d not found", *endNum)
+		}
+	}
+	return api.getModifiedAccounts(startBlock, endBlock)
+}
+
+// GetModifiedAccountsByHash returns all accounts that have changed between the
+// two blocks specified. A change is defined as a difference in nonce, balance,
+// code hash, or storage hash.
+//
+// With one parameter, returns the list of accounts modified in the specified block.
+func (api *PrivateDebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) {
+	var startBlock, endBlock *types.Block
+	startBlock = api.eth.blockchain.GetBlockByHash(startHash)
+	if startBlock == nil {
+		return nil, fmt.Errorf("start block %x not found", startHash)
+	}
+
+	if endHash == nil {
+		endBlock = startBlock
+		startBlock = api.eth.blockchain.GetBlockByHash(startBlock.ParentHash())
+		if startBlock == nil {
+			return nil, fmt.Errorf("block %x has no parent", endBlock.Number())
+		}
+	} else {
+		endBlock = api.eth.blockchain.GetBlockByHash(*endHash)
+		if endBlock == nil {
+			return nil, fmt.Errorf("end block %x not found", *endHash)
+		}
+	}
+	return api.getModifiedAccounts(startBlock, endBlock)
+}
+
+func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) {
+	if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
+		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
+	}
+
+	oldTrie, err := trie.NewSecure(startBlock.Root(), api.eth.chainDb, 0)
+	if err != nil {
+		return nil, err
+	}
+	newTrie, err := trie.NewSecure(endBlock.Root(), api.eth.chainDb, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
+	iter := trie.NewIterator(diff)
+
+	var dirty []common.Address
+	for iter.Next() {
+		key := newTrie.GetKey(iter.Key)
+		if key == nil {
+			return nil, fmt.Errorf("no preimage found for hash %x", iter.Key)
+		}
+		dirty = append(dirty, common.BytesToAddress(key))
+	}
+	return dirty, nil
+}

eth/backend.go

@@ -54,6 +54,7 @@ type LesServer interface {
 	Start(srvr *p2p.Server)
 	Stop()
 	Protocols() []p2p.Protocol
+	SetBloomBitsIndexer(bbIndexer *core.ChainIndexer)
 }
 
 // Ethereum implements the Ethereum full node service.
@@ -95,6 +96,7 @@ type Ethereum struct {
 
 func (s *Ethereum) AddLesServer(ls LesServer) {
 	s.lesServer = ls
+	ls.SetBloomBitsIndexer(s.bloomIndexer)
 }
 
 // New creates a new Ethereum object (including the
@@ -154,7 +156,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		eth.blockchain.SetHead(compat.RewindTo)
 		core.WriteChainConfig(chainDb, genesisHash, chainConfig)
 	}
-	eth.bloomIndexer.Start(eth.blockchain.CurrentHeader(), eth.blockchain.SubscribeChainEvent)
+	eth.bloomIndexer.Start(eth.blockchain)
 
 	if config.TxPool.Journal != "" {
 		config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal)
@@ -327,7 +329,7 @@ func (s *Ethereum) StartMining(local bool) error {
 		wallet, err := s.accountManager.Find(accounts.Account{Address: eb})
 		if wallet == nil || err != nil {
 			log.Error("Etherbase account unavailable locally", "err", err)
-			return fmt.Errorf("singer missing: %v", err)
+			return fmt.Errorf("signer missing: %v", err)
 		}
 		clique.Authorize(eb, wallet.SignHash)
 	}