params: release v1.8.6 to fix docker images

Dockerfile: revert the user change PR that broke all APIs

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sol linguist-language=Solidity

.travis.yml

@@ -12,34 +12,34 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: "1.10"
+      go: 1.10.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: "1.10"
+      go: 1.10.x
       script:
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - brew update
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: "1.10"
+      go: 1.10.x
       env:
         - lint
       git:
@@ -47,14 +47,12 @@ matrix:
       script:
         - go run build/ci.go lint
 
-    # This builder does the Ubuntu PPA and Linux Azure uploads
+    # This builder does the Ubuntu PPA upload
     - os: linux
       dist: trusty
-      sudo: required
-      go: "1.10"
+      go: 1.10.x
       env:
         - ubuntu-ppa
-        - azure-linux
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -63,11 +61,25 @@ matrix:
             - devscripts
             - debhelper
             - dput
-            - gcc-multilib
             - fakeroot
       script:
-        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+
+    # This builder does the Linux Azure uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.10.x
+      env:
+        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+      script:
+        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
         - go run build/ci.go install -arch 386
@@ -91,7 +103,7 @@ matrix:
       dist: trusty
       services:
         - docker
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-linux-mips
       git:
@@ -135,7 +147,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.10.1.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -152,7 +164,7 @@ matrix:
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-osx
         - azure-ios
@@ -181,8 +193,7 @@ matrix:
     # This builder does the Azure archive purges to avoid accumulating junk
     - os: linux
       dist: trusty
-      sudo: required
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-purge
       git:

Dockerfile

@@ -12,5 +12,5 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
-EXPOSE 8545 8546 30303 30303/udp 30304/udp
+EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -12,4 +12,4 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
 
-EXPOSE 8545 8546 30303 30303/udp 30304/udp
+EXPOSE 8545 8546 30303 30303/udp

Makefile

@@ -37,6 +37,9 @@ ios:
 test: all
 	build/env.sh go run build/ci.go test
 
+lint: ## Run linters.
+	build/env.sh go run build/ci.go lint
+
 clean:
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 

README.md

@@ -142,7 +142,7 @@ Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containe
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
-network via your own programs and not manually through the console. To aid this, Geth has built in
+network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
 exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).

VERSION

@@ -1 +1 @@
-1.8.3
+1.8.6

accounts/abi/numbers.go

@@ -25,23 +25,23 @@ import (
 )
 
 var (
-	big_t      = reflect.TypeOf(&big.Int{})
-	derefbig_t = reflect.TypeOf(big.Int{})
-	uint8_t    = reflect.TypeOf(uint8(0))
-	uint16_t   = reflect.TypeOf(uint16(0))
-	uint32_t   = reflect.TypeOf(uint32(0))
-	uint64_t   = reflect.TypeOf(uint64(0))
-	int_t      = reflect.TypeOf(int(0))
-	int8_t     = reflect.TypeOf(int8(0))
-	int16_t    = reflect.TypeOf(int16(0))
-	int32_t    = reflect.TypeOf(int32(0))
-	int64_t    = reflect.TypeOf(int64(0))
-	address_t  = reflect.TypeOf(common.Address{})
-	int_ts     = reflect.TypeOf([]int(nil))
-	int8_ts    = reflect.TypeOf([]int8(nil))
-	int16_ts   = reflect.TypeOf([]int16(nil))
-	int32_ts   = reflect.TypeOf([]int32(nil))
-	int64_ts   = reflect.TypeOf([]int64(nil))
+	bigT      = reflect.TypeOf(&big.Int{})
+	derefbigT = reflect.TypeOf(big.Int{})
+	uint8T    = reflect.TypeOf(uint8(0))
+	uint16T   = reflect.TypeOf(uint16(0))
+	uint32T   = reflect.TypeOf(uint32(0))
+	uint64T   = reflect.TypeOf(uint64(0))
+	intT      = reflect.TypeOf(int(0))
+	int8T     = reflect.TypeOf(int8(0))
+	int16T    = reflect.TypeOf(int16(0))
+	int32T    = reflect.TypeOf(int32(0))
+	int64T    = reflect.TypeOf(int64(0))
+	addressT  = reflect.TypeOf(common.Address{})
+	intTS     = reflect.TypeOf([]int(nil))
+	int8TS    = reflect.TypeOf([]int8(nil))
+	int16TS   = reflect.TypeOf([]int16(nil))
+	int32TS   = reflect.TypeOf([]int32(nil))
+	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
@@ -52,7 +52,7 @@ func U256(n *big.Int) []byte {
 // checks whether the given reflect value is signed. This also works for slices with a number type
 func isSigned(v reflect.Value) bool {
 	switch v.Type() {
-	case int_ts, int8_ts, int16_ts, int32_ts, int64_ts, int_t, int8_t, int16_t, int32_t, int64_t:
+	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
 		return true
 	}
 	return false

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbigT {
 		return indirect(v.Elem())
 	}
 	return v
@@ -36,26 +36,26 @@ func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type)
 	switch size {
 	case 8:
 		if unsigned {
-			return reflect.Uint8, uint8_t
+			return reflect.Uint8, uint8T
 		}
-		return reflect.Int8, int8_t
+		return reflect.Int8, int8T
 	case 16:
 		if unsigned {
-			return reflect.Uint16, uint16_t
+			return reflect.Uint16, uint16T
 		}
-		return reflect.Int16, int16_t
+		return reflect.Int16, int16T
 	case 32:
 		if unsigned {
-			return reflect.Uint32, uint32_t
+			return reflect.Uint32, uint32T
 		}
-		return reflect.Int32, int32_t
+		return reflect.Int32, int32T
 	case 64:
 		if unsigned {
-			return reflect.Uint64, uint64_t
+			return reflect.Uint64, uint64T
 		}
-		return reflect.Int64, int64_t
+		return reflect.Int64, int64T
 	}
-	return reflect.Ptr, big_t
+	return reflect.Ptr, bigT
 }
 
 // mustArrayToBytesSlice creates a new byte slice with the exact same size as value

accounts/abi/type.go

@@ -135,7 +135,7 @@ func NewType(t string) (typ Type, err error) {
 		typ.Type = reflect.TypeOf(bool(false))
 	case "address":
 		typ.Kind = reflect.Array
-		typ.Type = address_t
+		typ.Type = addressT
 		typ.Size = 20
 		typ.T = AddressTy
 	case "string":

accounts/abi/type_test.go

@@ -46,36 +46,36 @@ func TestTypeRegexp(t *testing.T) {
 		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
 		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
 		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
 		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
 		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
 		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
@@ -84,9 +84,9 @@ func TestTypeRegexp(t *testing.T) {
 		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
 		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
 		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -252,6 +252,9 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes20", common.Address{}, ""},
 		{"address", [20]byte{}, ""},
 		{"address", common.Address{}, ""},
+		{"bytes32[]]", "", "invalid arg type in abi"},
+		{"invalidType", "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil && len(test.err) == 0 {

accounts/abi/unpack_test.go

@@ -56,6 +56,23 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: true,
 	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000000",
+		want: false,
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000001000000000001",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000003",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
 	{
 		def:  `[{"type": "uint32"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",

accounts/url.go

@@ -74,6 +74,22 @@ func (u URL) MarshalJSON() ([]byte, error) {
 	return json.Marshal(u.String())
 }
 
+// UnmarshalJSON parses url.
+func (u *URL) UnmarshalJSON(input []byte) error {
+	var textUrl string
+	err := json.Unmarshal(input, &textUrl)
+	if err != nil {
+		return err
+	}
+	url, err := parseURL(textUrl)
+	if err != nil {
+		return err
+	}
+	u.Scheme = url.Scheme
+	u.Path = url.Path
+	return nil
+}
+
 // Cmp compares x and y and returns:
 //
 //   -1 if x <  y

accounts/usbwallet/hub.go

@@ -127,7 +127,7 @@ func (hub *Hub) refreshWallets() {
 		// breaking the Ledger protocol if that is waiting for user confirmation. This
 		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
 		// need to prevent concurrent comms ourselves. The more elegant solution would
-		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// be to ditch enumeration in favor of hotplug events, but that don't work yet
 		// on Windows so if we need to hack it anyway, this is more elegant for now.
 		hub.commsLock.Lock()
 		if hub.commsPend > 0 { // A confirmation is pending, don't refresh

accounts/usbwallet/ledger.go

@@ -53,11 +53,9 @@ const (
 	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
 
 	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
-	ledgerP1ConfirmFetchAddress     ledgerParam1 = 0x01 // Require a user confirmation before returning the address
 	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
 	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
 	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
-	ledgerP2ReturnAddressChainCode  ledgerParam2 = 0x01 // Require a user confirmation before returning the address
 )
 
 // errLedgerReplyInvalidHeader is the error message returned by a Ledger data exchange

accounts/usbwallet/wallet.go

@@ -99,7 +99,7 @@ type wallet struct {
 	//
 	// As such, a hardware wallet needs two locks to function correctly. A state
 	// lock can be used to protect the wallet's software-side internal state, which
-	// must not be held exlusively during hardware communication. A communication
+	// must not be held exclusively during hardware communication. A communication
 	// lock can be used to achieve exclusive access to the device itself, this one
 	// however should allow "skipping" waiting for operations that might want to
 	// use the device, but can live without too (e.g. account self-derivation).

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.10.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

bmt/bmt.go

@@ -75,7 +75,7 @@ type Hasher struct {
 	blocksize   int         // segment size (size of hash) also for hash.Hash
 	count       int         // segment count
 	size        int         // for hash.Hash same as hashsize
-	cur         int         // cursor position for righmost currently open chunk
+	cur         int         // cursor position for rightmost currently open chunk
 	segment     []byte      // the rightmost open segment (not complete)
 	depth       int         // index of last level
 	result      chan []byte // result channel
@@ -149,7 +149,7 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
 	}
 }
 
-// Drain drains the pool uptil it has no more than n resources
+// Drain drains the pool until it has no more than n resources
 func (self *TreePool) Drain(n int) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
@@ -412,11 +412,10 @@ func (self *Hasher) Reset() {
 
 // ResetWithLength needs to be called before writing to the hasher
 // the argument is supposed to be the byte slice binary representation of
-// the legth of the data subsumed under the hash
+// the length of the data subsumed under the hash
 func (self *Hasher) ResetWithLength(l []byte) {
 	self.Reset()
 	self.blockLength = l
-
 }
 
 // Release gives back the Tree to the pool whereby it unlocks
@@ -531,7 +530,7 @@ func (self *Hasher) finalise(n *Node, i int) (d int) {
 	for {
 		// when the final segment's path is going via left segments
 		// the incoming data is pushed to the parent upon pulling the left
-		// we do not need toogle the state since this condition is
+		// we do not need toggle the state since this condition is
 		// detectable
 		n.unbalanced = isLeft
 		n.right = nil

build/ci.go

@@ -329,7 +329,10 @@ func doLint(cmdline []string) {
 	// Run fast linters batched together
 	configs := []string{
 		"--vendor",
+		"--tests",
 		"--disable-all",
+		"--enable=goimports",
+		"--enable=varcheck",
 		"--enable=vet",
 		"--enable=gofmt",
 		"--enable=misspell",
@@ -340,7 +343,7 @@ func doLint(cmdline []string) {
 
 	// Run slow linters one by one
 	for _, linter := range []string{"unconvert", "gosimple"} {
-		configs = []string{"--vendor", "--deadline=10m", "--disable-all", "--enable=" + linter}
+		configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
 		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 	}
 }
@@ -766,7 +769,7 @@ func doAndroidArchive(cmdline []string) {
 		if meta.Develop {
 			repo = *deploy + "/content/repositories/snapshots"
 		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
+		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
 			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
 			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
 	}

build/goimports.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+find_files() {
+  find . -not \( \
+      \( \
+        -wholename '.github' \
+        -o -wholename './build/_workspace' \
+        -o -wholename './build/bin' \
+        -o -wholename './crypto/bn256' \
+        -o -wholename '*/vendor/*' \
+      \) -prune \
+    \) -name '*.go'
+}
+
+GOFMT="gofmt -s -w";
+GOIMPORTS="goimports -w";
+find_files | xargs $GOFMT;
+find_files | xargs $GOIMPORTS;

cmd/clef/README.md

@@ -0,0 +1,864 @@
+Clef
+----
+Clef can be used to sign transactions and data and is meant as a replacement for geth's account management.
+This allows DApps not to depend on geth's account management. When a DApp wants to sign data it can send the data to
+the signer, the signer will then provide the user with context and asks the user for permission to sign the data. If
+the users grants the signing request the signer will send the signature back to the DApp.
+  
+This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can
+help in situations when a DApp is connected to a remote node because a local Ethereum node is not available, not
+synchronised with the chain or a particular Ethereum node that has no built-in (or limited) account management.
+  
+Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
+or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
+
+
+## Command line flags
+Clef accepts the following command line options:
+```
+COMMANDS:
+   init    Initialize the signer, generate secret storage
+   attest  Attest that a js-file is to be used
+   addpw   Store a credential for a keystore file
+   help    Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --loglevel value        log level to emit to the screen (default: 4)
+   --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
+   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
+   --nousb                 Disables monitoring for and managing USB hardware wallets
+   --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcport value         HTTP-RPC server listening port (default: 8550)
+   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
+   --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
+   --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
+   --rules value           Enable rule-engine (default: "rules.json")
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --help, -h              show help
+   --version, -v           print the version
+
+```
+
+
+Example:
+```
+signer -keystore /my/keystore -chainid 4
+```
+
+Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works.
+
+## Security model
+
+The security model of the signer is as follows:
+
+* One critical component (the signer binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
+* The signer binary has a well-defined 'external' API.
+* The 'external' API is considered UNTRUSTED.
+* The signer binary also communicates with whatever process that invoked the binary, via stdin/stdout.
+  * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
+
+The general flow for signing a transaction using e.g. geth is as follows:
+![image](sign_flow.png)
+
+In this case, `geth` would be started with `--externalsigner=http://localhost:8550` and would relay requests to `eth.sendTransaction`.
+
+## TODOs
+
+Some snags and todos
+
+* [ ] The signer should take a startup param "--no-change", for UIs that do not contain the capability
+   to perform changes to things, only approve/deny. Such a UI should be able to start the signer in
+   a more secure mode by telling it that it only wants approve/deny capabilities.
+
+* [x] It would be nice if the signer could collect new 4byte-id:s/method selectors, and have a
+secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for
+inclusion upstream.
+
+* It should be possible to configure the signer to check if an account is indeed known to it, before
+passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate
+accounts if it immediately returned "unknown account".
+* [x] It should be possible to configure the signer to auto-allow listing (certain) accounts, instead of asking every time.
+* [x] Done Upon startup, the signer should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode),
+invoking methods with the following info:
+  * [x] Version info about the signer
+  * [x] Address of API (http/ipc)
+  * [ ] List of known accounts
+* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] `account_signRawTransaction`
+* [ ] `account_bulkSignTransactions([] transactions)` should
+   * only exist if enabled via config/flag
+   * only allow non-data-sending transactions
+   * all txs must use the same `from`-account
+   * let the user confirm, showing
+      * the total amount
+      * the number of unique recipients
+
+* Geth todos
+    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is
+put together is a bit of a hack into the http server. This could probably be greatly improved
+    - Relay: Geth should be started in `geth --external_signer localhost:8550`.
+    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This
+  type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which
+  retains the original input.
+    - The Geth api should switch to use the same type, and relay `to`-account verbatim to the external api.
+
+* [x] Storage
+    * [x] An encrypted key-value storage should be implemented
+    * See [rules.md](rules.md) for more info about this.
+
+* Another potential thing to introduce is pairing.
+  * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
+  * Thus geth/mist/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
+  * This feature would make the addition of rules less dangerous.
+
+* Wallets / accounts. Add API methods for wallets.
+
+## Communication
+
+### External API
+
+The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRPC standard as Geth. The messages are
+expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
+
+Some of these call can require user interaction. Clients must be aware that responses
+may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+
+The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
+that requests have any authority.
+
+### UI API
+
+The signer has one native console-based UI, for operation without any standalone tools.
+However, there is also an API to communicate with an external UI. To enable that UI,
+the signer needs to be executed with the `--stdio-ui` option, which allocates the
+`stdin`/`stdout` for the UI-api.
+
+An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
+
+The model is as follows:
+
+* The user starts the UI app (`pythonsigner.py`).
+* The UI app starts the `signer` with `--stdio-ui`, and listens to the
+process output for confirmation-requests.
+* The `signer` opens the external http api.
+* When the `signer` receives requests, it sends a `jsonrpc` request via `stdout`.
+* The UI app prompts the user accordingly, and responds to the `signer`
+* The `signer` signs (or not), and responds to the original request.
+
+## External API
+
+See the [external api changelog](extapi_changelog.md) for information about changes to this API.
+
+### Encoding
+- number: positive integers that are hex encoded
+- data: hex encoded data
+- string: ASCII string
+
+All hex encoded values must be prefixed with `0x`.
+
+## Methods
+
+### account_new
+
+#### Create new password protected account
+
+The signer will generate a new private key, encrypts it according to [web3 keystore spec](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) and stores it in the keystore directory.
+The client is responsible for creating a backup of the keystore. If the keystore is lost there is no method of retrieving lost accounts.
+
+#### Arguments
+
+None
+
+#### Result
+  - address [string]: account address that is derived from the generated key
+  - url [string]: location of the keyfile
+  
+#### Sample call
+```json
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "method": "account_new",
+  "params": []
+}
+
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+    "url": "keystore:///my/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+  }
+}
+```
+
+### account_list
+
+#### List available accounts
+   List all accounts that this signer currently manages
+
+#### Arguments
+
+None
+
+#### Result
+  - array with account records:
+     - account.address [string]: account address that is derived from the generated key
+     - account.type [string]: type of the 
+     - account.url [string]: location of the account
+  
+#### Sample call
+```json
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "method": "account_list"
+}
+
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "result": [
+    {
+      "address": "0xafb2f771f58513609765698f65d3f2f0224a956f",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T07-26-47.162109726Z--afb2f771f58513609765698f65d3f2f0224a956f"
+    },
+    {
+      "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+    }
+  ]
+}
+```
+
+### account_signTransaction
+
+#### Sign transactions
+   Signs a transactions and responds with the signed transaction in RLP encoded form.
+
+#### Arguments
+  2. transaction object:
+     - `from` [address]: account to send the transaction from
+     - `to` [address]: receiver account. If omitted or `0x`, will cause contract creation.
+     - `gas` [number]: maximum amount of gas to burn
+     - `gasPrice` [number]: gas price
+     - `value` [number:optional]: amount of Wei to send with the transaction
+     - `data` [data:optional]:  input data
+     - `nonce` [number]: account nonce
+  3. method signature [string:optional]
+     - The method signature, if present, is to aid decoding the calldata. Should consist of `methodname(paramtype,...)`, e.g. `transfer(uint256,address)`. The signer may use this data to parse the supplied calldata, and show the user. The data, however, is considered totally untrusted, and reliability is not expected.
+
+
+#### Result
+  - signed transaction in RLP encoded form [data]
+  
+#### Sample call
+```json
+{
+  "id": 2,
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+      "gas": "0x55555",
+      "gasPrice": "0x1234",
+      "input": "0xabcd",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x1234"
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "error": {
+    "code": -32000,
+    "message": "Request denied"
+  }
+}
+```
+#### Sample call with ABI-data
+
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+      "gas": "0x333",
+      "gasPrice": "0x1",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
+    },
+    "safeSend(address)"
+  ],
+  "id": 67
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "result": {
+    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+    "tx": {
+      "nonce": "0x0",
+      "gasPrice": "0x1",
+      "gas": "0x333",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+      "v": "0x26",
+      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
+      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
+    }
+  }
+}
+```
+
+Bash example:
+```bash
+#curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+
+{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
+```
+
+
+### account_sign
+
+#### Sign data
+   Signs a chunk of data and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [data]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "method": "account_sign",
+  "params": [
+    "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+    "0xaabbccdd"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "result": "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+}
+```
+
+### account_ecRecover
+
+#### Recover address
+   Derive the address from the account that was used to sign data from the data and signature.
+   
+#### Arguments
+  - data [data]: data that was signed
+  - signature [data]: the signature to verify
+
+#### Result
+  - derived account [address]
+  
+#### Sample call
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "method": "account_ecRecover",
+  "params": [
+    "0xaabbccdd",
+    "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
+}
+
+```
+
+### account_import
+
+#### Import account
+   Import a private key into the keystore. The imported key is expected to be encrypted according to the web3 keystore
+   format.
+   
+#### Arguments
+  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export) 
+
+#### Result
+  - imported key [object]:
+     - key.address [address]: address of the imported key
+     - key.type [string]: type of the account
+     - key.url [string]: key URL
+  
+#### Sample call
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "method": "account_import",
+  "params": [
+    {
+      "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+      "crypto": {
+        "cipher": "aes-128-ctr",
+        "cipherparams": {
+          "iv": "401c39a7c7af0388491c3d3ecb39f532"
+        },
+        "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+        "kdf": "scrypt",
+        "kdfparams": {
+          "dklen": 32,
+          "n": 262144,
+          "p": 1,
+          "r": 8,
+          "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+        },
+        "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+      },
+      "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+      "version": 3
+    },
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xc7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "type": "account",
+    "url": "keystore:///tmp/keystore/UTC--2017-08-24T11-00-42.032024108Z--c7412fc59930fd90099c917a50e5f11d0934b2f5"
+  }
+}
+```
+
+### account_export
+
+#### Export account from keystore
+   Export a private key from the keystore. The exported private key is encrypted with the original passphrase. When the
+   key is imported later this passphrase is required.
+   
+#### Arguments
+  - account [address]: export private key that is associated with this account
+
+#### Result
+  - exported key, see [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) for
+  more information
+  
+#### Sample call
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "method": "account_export",
+  "params": [
+    "0xc7412fc59930fd90099c917a50e5f11d0934b2f5"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "crypto": {
+      "cipher": "aes-128-ctr",
+      "cipherparams": {
+        "iv": "401c39a7c7af0388491c3d3ecb39f532"
+      },
+      "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+      "kdf": "scrypt",
+      "kdfparams": {
+        "dklen": 32,
+        "n": 262144,
+        "p": 1,
+        "r": 8,
+        "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+      },
+      "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+    },
+    "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+    "version": 3
+  }
+}
+```
+
+
+
+## UI API
+
+These methods needs to be implemented by a UI listener.
+
+By starting the signer with the switch `--stdio-ui-test`, the signer will invoke all known methods, and expect the UI to respond with
+denials. This can be used during development to ensure that the API is (at least somewhat) correctly implemented.
+See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to perform the 'denial-handshake-test'.
+
+All methods in this API uses object-based parameters, so that there can be no mixups of parameters: each piece of data is accessed by key.
+
+See the [ui api changelog](intapi_changelog.md) for information about changes to this API.
+
+OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
+Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
+things simpler for both parties.
+
+### ApproveTx
+
+Invoked when there's a transaction for approval.
+
+
+#### Sample call
+
+Here's a method invocation:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "Info",
+            "message": "safeSend(address: 0x0000000000000000000000000000000000000012)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48486",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+```
+
+The same method invocation, but with invalid data:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000002000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000002000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "WARNING",
+            "message": "Transaction data did not match ABI-interface: WARNING: Supplied data is stuffed with extra data. \nWant 0000000000000002000000000000000000000000000000000000000000000012\nHave 0000000000000000000000000000000000000000000000000000000000000012\nfor method safeSend(address)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48492",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+
+```
+
+One which has missing `to`, but with no `data`:
+
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 3,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "",
+        "to": null,
+        "gas": "0x0",
+        "gasPrice": "0x0",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": null,
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "CRITICAL",
+            "message": "Tx will create contract with empty code!"
+          }
+        ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+```
+
+### ApproveExport
+
+Invoked when a request to export an account has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 7,
+  "method": "ApproveExport",
+  "params": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ApproveListing
+
+Invoked when a request for account listing has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 5,
+  "method": "ApproveListing",
+  "params": [
+    {
+      "accounts": [
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-20T14-44-54.089682944Z--123409812340981234098123409812deadbeef42",
+          "address": "0x123409812340981234098123409812deadbeef42"
+        },
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-23T21-59-03.199240693Z--cafebabedeadbeef34098123409812deadbeef42",
+          "address": "0xcafebabedeadbeef34098123409812deadbeef42"
+        }
+      ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+
+### ApproveSignData
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 4,
+  "method": "ApproveSignData",
+  "params": [
+    {
+      "address": "0x123409812340981234098123409812deadbeef42",
+      "raw_data": "0x01020304",
+      "message": "\u0019Ethereum Signed Message:\n4\u0001\u0002\u0003\u0004",
+      "hash": "0x7e3a4e7a9d1744bc5c675c25e1234ca8ed9162bd17f78b9085e48047c15ac310",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ShowInfo
+
+The UI should show the info to the user. Does not expect response.
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 9,
+  "method": "ShowInfo",
+  "params": [
+    {
+      "text": "Tests completed"
+    }
+  ]
+}
+
+```
+
+### ShowError
+
+The UI should show the info to the user. Does not expect response.
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ShowError",
+  "params": [
+    {
+      "text": "Testing 'ShowError'"
+    }
+  ]
+}
+
+```
+
+### OnApproved
+
+`OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
+
+When implementing rate-limited rules, this callback should be used.
+
+TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
+
+### OnSignerStartup
+
+This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
+in k/v-form.
+
+Example call:
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+
+```
+
+
+### Rules for UI apis
+
+A UI should conform to the following rules.
+
+* A UI MUST NOT load any external resources that were not embedded/part of the UI package.
+  * For example, not load icons, stylesheets from the internet
+  * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
+* A Graphical UI MUST show the blocky-identicon for ethereum addresses.
+* A UI MUST warn display approproate warning if the destination-account is formatted with invalid checksum.
+* A UI MUST NOT open any ports or services
+  * The signer opens the public port
+* A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
+* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
+* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
+  * The signer provides accounts
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+along with the UI.
+
+

cmd/clef/extapi_changelog.md

@@ -0,0 +1,25 @@
+### Changelog for external API
+
+
+
+#### 2.0.0
+
+* Commit `73abaf04b1372fa4c43201fb1b8019fe6b0a6f8d`, move `from` into `transaction` object in `signTransaction`. This
+makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
+
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/intapi_changelog.md

@@ -0,0 +1,86 @@
+### Changelog for internal API (ui-api)
+
+### 2.0.0
+
+* Modify how `call_info` on a transaction is conveyed. New format:
+
+```
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x82A2A876D39022B3019932D30Cd9c97ad5616813",
+        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x123",
+        "value": "0x10",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+        {
+          "type": "WARNING",
+          "message": "Invalid checksum on to-address"
+        },
+        {
+          "type": "WARNING",
+          "message": "Tx contains data, but provided ABI signature could not be matched: Did not match: test (0 matches)"
+        }
+      ],
+      "meta": {
+        "remote": "127.0.0.1:54286",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+```
+
+#### 1.2.0
+
+* Add `OnStartup` method, to provide the UI with information about what API version
+the signer uses (both internal and external) aswell as build-info and external api.
+
+Example call:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+```
+
+#### 1.1.0
+
+* Add `OnApproved` method
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/main.go

@@ -0,0 +1,640 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// signer is a utility that can be used so sign transactions and
+// arbitrary data.
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/rules"
+	"github.com/ethereum/go-ethereum/signer/storage"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// ExternalApiVersion -- see extapi_changelog.md
+const ExternalApiVersion = "2.0.0"
+
+// InternalApiVersion -- see intapi_changelog.md
+const InternalApiVersion = "2.0.0"
+
+const legalWarning = `
+WARNING! 
+
+Clef is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing. 
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE! 
+
+`
+
+var (
+	logLevelFlag = cli.IntFlag{
+		Name:  "loglevel",
+		Value: 4,
+		Usage: "log level to emit to the screen",
+	}
+	keystoreFlag = cli.StringFlag{
+		Name:  "keystore",
+		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
+		Usage: "Directory for the keystore",
+	}
+	configdirFlag = cli.StringFlag{
+		Name:  "configdir",
+		Value: DefaultConfigDir(),
+		Usage: "Directory for Clef configuration",
+	}
+	rpcPortFlag = cli.IntFlag{
+		Name:  "rpcport",
+		Usage: "HTTP-RPC server listening port",
+		Value: node.DefaultHTTPPort + 5,
+	}
+	signerSecretFlag = cli.StringFlag{
+		Name:  "signersecret",
+		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+	}
+	dBFlag = cli.StringFlag{
+		Name:  "4bytedb",
+		Usage: "File containing 4byte-identifiers",
+		Value: "./4byte.json",
+	}
+	customDBFlag = cli.StringFlag{
+		Name:  "4bytedb-custom",
+		Usage: "File used for writing new 4byte-identifiers submitted via API",
+		Value: "./4byte-custom.json",
+	}
+	auditLogFlag = cli.StringFlag{
+		Name:  "auditlog",
+		Usage: "File used to emit audit logs. Set to \"\" to disable",
+		Value: "audit.log",
+	}
+	ruleFlag = cli.StringFlag{
+		Name:  "rules",
+		Usage: "Enable rule-engine",
+		Value: "rules.json",
+	}
+	stdiouiFlag = cli.BoolFlag{
+		Name: "stdio-ui",
+		Usage: "Use STDIN/STDOUT as a channel for an external UI. " +
+			"This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " +
+			"interface, and can be used when Clef is started by an external process.",
+	}
+	testFlag = cli.BoolFlag{
+		Name:  "stdio-ui-test",
+		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
+	}
+	app         = cli.NewApp()
+	initCommand = cli.Command{
+		Action:    utils.MigrateFlags(initializeSecrets),
+		Name:      "init",
+		Usage:     "Initialize the signer, generate secret storage",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+		},
+		Description: `
+The init command generates a master seed which Clef can use to store credentials and data needed for 
+the rule-engine to work.`,
+	}
+	attestCommand = cli.Command{
+		Action:    utils.MigrateFlags(attestFile),
+		Name:      "attest",
+		Usage:     "Attest that a js-file is to be used",
+		ArgsUsage: "<sha256sum>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The attest command stores the sha256 of the rule.js-file that you want to use for automatic processing of 
+incoming requests. 
+
+Whenever you make an edit to the rule file, you need to use attestation to tell 
+Clef that the file is 'safe' to execute.`,
+	}
+
+	addCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(addCredential),
+		Name:      "addpw",
+		Usage:     "Store a credential for a keystore file",
+		ArgsUsage: "<address> <password>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+remove any stored credential for that address (keyfile)
+`,
+	}
+)
+
+func init() {
+	app.Name = "Clef"
+	app.Usage = "Manage Ethereum account operations"
+	app.Flags = []cli.Flag{
+		logLevelFlag,
+		keystoreFlag,
+		configdirFlag,
+		utils.NetworkIdFlag,
+		utils.LightKDFFlag,
+		utils.NoUSBFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCVirtualHostsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+		utils.RPCEnabledFlag,
+		rpcPortFlag,
+		signerSecretFlag,
+		dBFlag,
+		customDBFlag,
+		auditLogFlag,
+		ruleFlag,
+		stdiouiFlag,
+		testFlag,
+	}
+	app.Action = signer
+	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+
+}
+func main() {
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func initializeSecrets(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	configDir := c.String(configdirFlag.Name)
+
+	masterSeed := make([]byte, 256)
+	n, err := io.ReadFull(rand.Reader, masterSeed)
+	if err != nil {
+		return err
+	}
+	if n != len(masterSeed) {
+		return fmt.Errorf("failed to read enough random")
+	}
+	err = os.Mkdir(configDir, 0700)
+	if err != nil && !os.IsExist(err) {
+		return err
+	}
+	location := filepath.Join(configDir, "secrets.dat")
+	if _, err := os.Stat(location); err == nil {
+		return fmt.Errorf("file %v already exists, will not overwrite", location)
+	}
+	err = ioutil.WriteFile(location, masterSeed, 0700)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("A master seed has been generated into %s\n", location)
+	fmt.Printf(`
+This is required to be able to store credentials, such as : 
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it. 
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+
+`)
+	return nil
+}
+func attestFile(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+	// Initialize the encrypted storages
+	configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey)
+	val := ctx.Args().First()
+	configStorage.Put("ruleset_sha256", val)
+	log.Info("Ruleset attestation updated", "sha256", val)
+	return nil
+}
+
+func addCredential(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires at leaste one argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+
+	// Initialize the encrypted storages
+	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	key := ctx.Args().First()
+	value := ""
+	if len(ctx.Args()) > 1 {
+		value = ctx.Args().Get(1)
+	}
+	pwStorage.Put(key, value)
+	log.Info("Credential store updated", "key", key)
+	return nil
+}
+
+func initialize(c *cli.Context) error {
+	// Set up the logger to print everything
+	logOutput := os.Stdout
+	if c.Bool(stdiouiFlag.Name) {
+		logOutput = os.Stderr
+		// If using the stdioui, we can't do the 'confirm'-flow
+		fmt.Fprintf(logOutput, legalWarning)
+	} else {
+		if !confirm(legalWarning) {
+			return fmt.Errorf("aborted by user")
+		}
+	}
+
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(logOutput, log.TerminalFormat(true))))
+	return nil
+}
+
+func signer(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	var (
+		ui core.SignerUI
+	)
+	if c.Bool(stdiouiFlag.Name) {
+		log.Info("Using stdin/stdout as UI-channel")
+		ui = core.NewStdIOUI()
+	} else {
+		log.Info("Using CLI as UI-channel")
+		ui = core.NewCommandlineUI()
+	}
+	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+
+	var (
+		api core.ExternalAPI
+	)
+
+	configDir := c.String(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c); err != nil {
+		log.Info("No master seed provided, rules disabled")
+	} else {
+
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+
+		// Generate domain specific keys
+		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
+		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+		// Initialize the encrypted storages
+		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
+		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+
+		//Do we have a rule-file?
+		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		if err != nil {
+			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
+		} else {
+			hasher := sha256.New()
+			hasher.Write(ruleJS)
+			shasum := hasher.Sum(nil)
+			storedShasum := configStorage.Get("ruleset_sha256")
+			if storedShasum != hex.EncodeToString(shasum) {
+				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+			} else {
+				// Initialize rules
+				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
+				if err != nil {
+					utils.Fatalf(err.Error())
+				}
+				ruleEngine.Init(string(ruleJS))
+				ui = ruleEngine
+				log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
+			}
+		}
+	}
+
+	apiImpl := core.NewSignerAPI(
+		c.Int64(utils.NetworkIdFlag.Name),
+		c.String(keystoreFlag.Name),
+		c.Bool(utils.NoUSBFlag.Name),
+		ui, db,
+		c.Bool(utils.LightKDFFlag.Name))
+
+	api = apiImpl
+
+	// Audit logging
+	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+		api, err = core.NewAuditLogger(logfile, api)
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		log.Info("Audit logs configured", "file", logfile)
+	}
+	// register signer API with server
+	var (
+		extapiUrl = "n/a"
+		ipcApiUrl = "n/a"
+	)
+	rpcApi := []rpc.API{
+		{
+			Namespace: "account",
+			Public:    true,
+			Service:   api,
+			Version:   "1.0"},
+	}
+	if c.Bool(utils.RPCEnabledFlag.Name) {
+
+		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
+		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
+
+		// start http server
+		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts)
+		if err != nil {
+			utils.Fatalf("Could not start RPC api: %v", err)
+		}
+		extapiUrl = fmt.Sprintf("http://%s", httpEndpoint)
+		log.Info("HTTP endpoint opened", "url", extapiUrl)
+
+		defer func() {
+			listener.Close()
+			log.Info("HTTP endpoint closed", "url", httpEndpoint)
+		}()
+
+	}
+	if !c.Bool(utils.IPCDisabledFlag.Name) {
+		if c.IsSet(utils.IPCPathFlag.Name) {
+			ipcApiUrl = c.String(utils.IPCPathFlag.Name)
+		} else {
+			ipcApiUrl = filepath.Join(configDir, "clef.ipc")
+		}
+
+		listener, _, err := rpc.StartIPCEndpoint(ipcApiUrl, rpcApi)
+		if err != nil {
+			utils.Fatalf("Could not start IPC api: %v", err)
+		}
+		log.Info("IPC endpoint opened", "url", ipcApiUrl)
+		defer func() {
+			listener.Close()
+			log.Info("IPC endpoint closed", "url", ipcApiUrl)
+		}()
+
+	}
+
+	if c.Bool(testFlag.Name) {
+		log.Info("Performing UI test")
+		go testExternalUI(apiImpl)
+	}
+	ui.OnSignerStartup(core.StartupInfo{
+		Info: map[string]interface{}{
+			"extapi_version": ExternalApiVersion,
+			"intapi_version": InternalApiVersion,
+			"extapi_http":    extapiUrl,
+			"extapi_ipc":     ipcApiUrl,
+		},
+	})
+
+	abortChan := make(chan os.Signal)
+	signal.Notify(abortChan, os.Interrupt)
+
+	sig := <-abortChan
+	log.Info("Exiting...", "signal", sig)
+
+	return nil
+}
+
+// splitAndTrim splits input separated by a comma
+// and trims excessive white space from the substrings.
+func splitAndTrim(input string) []string {
+	result := strings.Split(input, ",")
+	for i, r := range result {
+		result[i] = strings.TrimSpace(r)
+	}
+	return result
+}
+
+// DefaultConfigDir is the default config directory to use for the vaults and other
+// persistence requirements.
+func DefaultConfigDir() string {
+	// Try to place the data folder in the user's home dir
+	home := homeDir()
+	if home != "" {
+		if runtime.GOOS == "darwin" {
+			return filepath.Join(home, "Library", "Signer")
+		} else if runtime.GOOS == "windows" {
+			return filepath.Join(home, "AppData", "Roaming", "Signer")
+		} else {
+			return filepath.Join(home, ".clef")
+		}
+	}
+	// As we cannot guess a stable location, return empty and handle later
+	return ""
+}
+
+func homeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+	return ""
+}
+func readMasterKey(ctx *cli.Context) ([]byte, error) {
+	var (
+		file      string
+		configDir = ctx.String(configdirFlag.Name)
+	)
+	if ctx.IsSet(signerSecretFlag.Name) {
+		file = ctx.String(signerSecretFlag.Name)
+	} else {
+		file = filepath.Join(configDir, "secrets.dat")
+	}
+	if err := checkFile(file); err != nil {
+		return nil, err
+	}
+	masterKey, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	if len(masterKey) < 256 {
+		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	}
+	// Create vault location
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	err = os.Mkdir(vaultLocation, 0700)
+	if err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	//!TODO, use KDF to stretch the master key
+	//			stretched_key := stretch_key(master_key)
+
+	return masterKey, nil
+}
+
+// checkFile is a convenience function to check if a file
+// * exists
+// * is mode 0600
+func checkFile(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("failed stat on %s: %v", filename, err)
+	}
+	// Check the unix permission bits
+	if info.Mode().Perm()&077 != 0 {
+		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
+	}
+	return nil
+}
+
+// confirm displays a text and asks for user confirmation
+func confirm(text string) bool {
+	fmt.Printf(text)
+	fmt.Printf("\nEnter 'ok' to proceed:\n>")
+
+	text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+
+	if text := strings.TrimSpace(text); text == "ok" {
+		return true
+	}
+	return false
+}
+
+func testExternalUI(api *core.SignerAPI) {
+
+	ctx := context.WithValue(context.Background(), "remote", "clef binary")
+	ctx = context.WithValue(ctx, "scheme", "in-proc")
+	ctx = context.WithValue(ctx, "local", "main")
+
+	errs := make([]string, 0)
+
+	api.UI.ShowInfo("Testing 'ShowInfo'")
+	api.UI.ShowError("Testing 'ShowError'")
+
+	checkErr := func(method string, err error) {
+		if err != nil && err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+		}
+	}
+	var err error
+
+	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
+	checkErr("SignTransaction", err)
+	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+	checkErr("Sign", err)
+	_, err = api.List(ctx)
+	checkErr("List", err)
+	_, err = api.New(ctx)
+	checkErr("New", err)
+	_, err = api.Export(ctx, common.Address{})
+	checkErr("Export", err)
+	_, err = api.Import(ctx, json.RawMessage{})
+	checkErr("Import", err)
+
+	api.UI.ShowInfo("Tests completed")
+
+	if len(errs) > 0 {
+		log.Error("Got errors")
+		for _, e := range errs {
+			log.Error(e)
+		}
+	} else {
+		log.Info("No errors")
+	}
+
+}
+
+/**
+//Create Account
+
+curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_new","params":["test"],"id":67}' localhost:8550
+
+// List accounts
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_list","params":[""],"id":67}' http://localhost:8550/
+
+// Make Transaction
+// safeSend(0x12)
+// 4401a6e40000000000000000000000000000000000000000000000000000000000000012
+
+// supplied abi
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"test"],"id":67}' http://localhost:8550/
+
+// Not supplied
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"}],"id":67}' http://localhost:8550/
+
+// Sign data
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_sign","params":["0x694267f14675d7e1b9494fd8d72fefe1755710fa","bazonk gaz baz"],"id":67}' http://localhost:8550/
+
+
+**/

cmd/clef/pythonsigner.py

@@ -0,0 +1,179 @@
+import os,sys, subprocess
+from tinyrpc.transports import ServerTransport
+from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
+from tinyrpc.dispatch import public,RPCDispatcher
+from tinyrpc.server import RPCServer
+
+""" This is a POC example of how to write a custom UI for Clef. The UI starts the
+clef process with the '--stdio-ui' option, and communicates with clef using standard input / output.
+
+The standard input/output is a relatively secure way to communicate, as it does not require opening any ports
+or IPC files. Needless to say, it does not protect against memory inspection mechanisms where an attacker
+can access process memory."""
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib as urlparse
+
+class StdIOTransport(ServerTransport):
+    """ Uses std input/output for RPC """
+    def receive_message(self):
+        return None, urlparse.unquote(sys.stdin.readline())
+
+    def send_reply(self, context, reply):
+        print(reply)
+
+class PipeTransport(ServerTransport):
+    """ Uses std a pipe for RPC """
+
+    def __init__(self,input, output):
+        self.input = input
+        self.output = output
+
+    def receive_message(self):
+        data = self.input.readline()
+        print(">> {}".format( data))
+        return None, urlparse.unquote(data)
+
+    def send_reply(self, context, reply):
+        print("<< {}".format( reply))
+        self.output.write(reply)
+        self.output.write("\n")
+
+class StdIOHandler():
+
+    def __init__(self):
+        pass
+
+    @public
+    def ApproveTx(self,req):
+        """
+        Example request:
+        {
+            "jsonrpc": "2.0",
+            "method": "ApproveTx",
+            "params": [{
+                "transaction": {
+                    "to": "0xae967917c465db8578ca9024c205720b1a3651A9",
+                    "gas": "0x333",
+                    "gasPrice": "0x123",
+                    "value": "0x10",
+                    "data": "0xd7a5865800000000000000000000000000000000000000000000000000000000000000ff",
+                    "nonce": "0x0"
+                },
+                "from": "0xAe967917c465db8578ca9024c205720b1a3651A9",
+                "call_info": "Warning! Could not validate ABI-data against calldata\nSupplied ABI spec does not contain method signature in data: 0xd7a58658",
+                "meta": {
+                    "remote": "127.0.0.1:34572",
+                    "local": "localhost:8550",
+                    "scheme": "HTTP/1.1"
+                }
+            }],
+            "id": 1
+        }
+
+        :param transaction: transaction info
+        :param call_info: info abou the call, e.g. if ABI info could not be
+        :param meta: metadata about the request, e.g. where the call comes from
+        :return: 
+        """
+        transaction = req.get('transaction')
+        _from       = req.get('from')
+        call_info   = req.get('call_info')
+        meta        = req.get('meta')
+
+        return {
+            "approved" : False,
+            #"transaction" : transaction,
+  #          "from" : _from,
+#            "password" : None,
+        }
+
+    @public
+    def ApproveSignData(self, req):
+        """ Example request
+
+        """
+        return {"approved": False, "password" : None}
+
+    @public
+    def ApproveExport(self, req):
+        """ Example request
+
+        """
+        return {"approved" : False}
+
+    @public
+    def ApproveImport(self, req):
+        """ Example request
+
+        """
+        return { "approved" : False, "old_password": "", "new_password": ""}
+
+    @public
+    def ApproveListing(self, req):
+        """ Example request
+
+        """
+        return {'accounts': []}
+
+    @public
+    def ApproveNewAccount(self, req):
+        """
+        Example request
+
+        :return:
+        """
+        return {"approved": False,
+                #"password": ""
+                }
+
+    @public
+    def ShowError(self,message = {}):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowError'"},"id":1}
+
+        :param message: to show
+        :return: nothing
+        """
+        if 'text' in message.keys():
+            sys.stderr.write("Error: {}\n".format( message['text']))
+        return
+
+    @public
+    def ShowInfo(self,message = {}):
+        """
+        Example request
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowInfo'"},"id":0}
+
+        :param message: to display
+        :return:nothing
+        """
+
+        if 'text' in message.keys():
+            sys.stdout.write("Error: {}\n".format( message['text']))
+        return
+
+def main(args):
+
+    cmd = ["./clef", "--stdio-ui"]
+    if len(args) > 0 and args[0] == "test":
+        cmd.extend(["--stdio-ui-test"])
+    print("cmd: {}".format(" ".join(cmd)))
+    dispatcher = RPCDispatcher()
+    dispatcher.register_instance(StdIOHandler(), '')
+    # line buffered
+    p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    rpc_server = RPCServer(
+        PipeTransport(p.stdout, p.stdin),
+        JSONRPCProtocol(),
+        dispatcher
+    )
+    rpc_server.serve_forever()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

cmd/clef/rules.md

@@ -0,0 +1,236 @@
+# Rules
+
+The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
+
+It enables usecases like the following:
+
+* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
+* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`
+
+The two main features that are required for this to work well are;
+
+1. Rule Implementation: how to create, manage and interpret rules in a flexible but secure manner
+2. Credential managements and credentials; how to provide auto-unlock without exposing keys unnecessarily.
+
+The section below deals with both of them
+
+## Rule Implementation
+
+A ruleset file is implemented as a `js` file. Under the hood, the ruleset-engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
+defined in the UI protocol. Example:
+
+```javascript
+
+function asBig(str){
+    if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+    return new BigNumber(str)
+}
+
+// Approve transactions to a certain contract if value is below a certain limit
+function ApproveTx(req){
+
+    var limit = big.Newint("0xb1a2bc2ec50000")
+	var value = asBig(req.transaction.value);
+
+	if(req.transaction.to.toLowerCase()=="0xae967917c465db8578ca9024c205720b1a3651a9")
+	    && value.lt(limit) ){
+	    return "Approve"
+	 }
+    // If we return "Reject", it will be rejected.
+    // By not returning anything, it will be passed to the next UI, for manual processing
+}
+
+//Approve listings if request made from IPC
+function ApproveListing(req){
+    if (req.metadata.scheme == "ipc"){ return "Approve"}
+}
+
+```
+
+Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
+invokes the corresponding method. In doing so, there are three possible outcomes:
+
+1. JS returns "Approve"
+  * Auto-approve request
+2. JS returns "Reject"
+  * Auto-reject request
+3. Error occurs, or something else is returned
+  * Pass on to `next` ui: the regular UI channel.
+
+A more advanced example can be found below, "Example 1: ruleset for a rate-limited window", using `storage` to `Put` and `Get` `string`s by key.
+
+* At the time of writing, storage only exists as an ephemeral unencrypted implementation, to be used during testing.
+
+### Things to note
+
+The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
+
+* "use strict" will parse, but does nothing.
+* The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification.
+* Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported.
+
+Additionally, a few more have been added
+
+* The rule execution cannot load external javascript files.
+* The only preloaded libary is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
+* Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
+* Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
+* The JS engine has access to `storage` and `console`.
+
+#### Security considerations
+
+##### Security of ruleset
+
+Some security precautions can be made, such as:
+
+* Never load `ruleset.js` unless the file is `readonly` (`r-??-??-?`). If the user wishes to modify the ruleset, he must make it writeable and then set back to readonly.
+  * This is to prevent attacks where files are dropped on the users disk.
+* Since we're going to have to have some form of secure storage (not defined in this section), we could also store the `sha3` of the `ruleset.js` file in there.
+  * If the user wishes to modify the ruleset, he'd then have to perform e.g. `signer --attest /path/to/ruleset --credential <creds>`
+
+##### Security of implementation
+
+The drawbacks of this very flexible solution is that the `signer` needs to contain a javascript engine. This is pretty simple to implement, since it's already
+implemented for `geth`. There are no known security vulnerabilities in, nor have we had any security-problems with it so far.
+
+The javascript engine would be an added attack surface; but if the validation of `rulesets` is made good (with hash-based attestation), the actual javascript cannot be considered
+an attack surface -- if an attacker can control the ruleset, a much simpler attack would be to implement an "always-approve" rule instead of exploiting the js vm. The only benefit
+to be gained from attacking the actual `signer` process from the `js` side would be if it could somehow extract cryptographic keys from memory.
+
+##### Security in usability
+
+Javascript is flexible, but also easy to get wrong, especially when users assume that `js` can handle large integers natively. Typical errors
+include trying to multiply `gasCost` with `gas` without using `bigint`:s.
+
+It's unclear whether any other DSL could be more secure; since there's always the possibility of erroneously implementing a rule.
+
+
+## Credential management
+
+The ability to auto-approve transaction means that the signer needs to have necessary credentials to decrypt keyfiles. These passwords are hereafter called `ksp` (keystore pass).
+
+### Example implementation
+
+Upon startup of the signer, the signer is given a switch: `--seed <path/to/masterseed>`
+The `seed` contains a blob of bytes, which is the master seed for the `signer`.
+
+The `signer` uses the `seed` to:
+
+* Generate the `path` where the settings are stored.
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/vault.dat`
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/rules.js`
+* Generate the encryption password for `vault.dat`.
+
+The `vault.dat` would be an encrypted container storing the following information:
+
+* `ksp` entries
+* `sha256` hash of `rules.js`
+* Information about pair:ed callers (not yet specified)
+
+### Security considerations
+
+This would leave it up to the user to ensure that the `path/to/masterseed` is handled in a secure way. It's difficult to get around this, although one could
+imagine leveraging OS-level keychains where supported. The setup is however in general similar to how ssh-keys are  stored in `.ssh/`.
+
+
+# Implementation status
+
+This is now implemented (with ephemeral non-encrypted storage for now, so not yet enabled).
+
+## Example 1: ruleset for a rate-limited window
+
+
+```javascript
+
+	function big(str){
+		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str)
+	}
+
+	// Time window: 1 week
+	var window = 1000* 3600*24*7;
+
+	// Limit : 1 ether
+	var limit = new BigNumber("1e18");
+
+	function isLimitOk(transaction){
+		var value = big(transaction.value)
+		// Start of our window function
+		var windowstart = new Date().getTime() - window;
+
+		var txs = [];
+		var stored = storage.Get('txs');
+
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// First, remove all that have passed out of the time-window
+		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
+		console.log(txs, newtxs.length);
+
+		// Secondly, aggregate the current sum
+		sum = new BigNumber(0)
+
+		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+		console.log("ApproveTx > Sum so far", sum);
+		console.log("ApproveTx > Requested", value.toNumber());
+
+		// Would we exceed weekly limit ?
+		return sum.plus(value).lt(limit)
+
+	}
+	function ApproveTx(r){
+		if (isLimitOk(r.transaction)){
+			return "Approve"
+		}
+		return "Nope"
+	}
+
+	/**
+	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+ 	* 'response_str' contains the return value that will be sent to the external caller.
+	* The return value from this method is ignore - the reason for having this callback is to allow the
+	* ruleset to keep track of approved transactions.
+	*
+	* When implementing rate-limited rules, this callback should be used.
+	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+	* then accepts the transaction, this method will be called.
+	*
+	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+	*/
+ 	function OnApprovedTx(resp){
+		var value = big(resp.tx.value)
+		var txs = []
+		// Load stored transactions
+		var stored = storage.Get('txs');
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// Add this to the storage
+		txs.push({tstamp: new Date().getTime(), value: value});
+		storage.Put("txs", JSON.stringify(txs));
+	}
+
+```
+
+## Example 2: allow destination
+
+```javascript
+
+	function ApproveTx(r){
+		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
+		// Otherwise goes to manual processing
+	}
+
+```
+
+## Example 3: Allow listing
+
+```javascript
+
+    function ApproveListing(){
+        return "Approve"
+    }
+
+```

cmd/clef/tutorial.md

@@ -0,0 +1,198 @@
+## Initializing the signer
+
+First, initialize the master seed.
+
+```text
+#./signer init
+
+WARNING!
+
+The signer is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing.
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE!
+
+
+Enter 'ok' to proceed:
+>ok
+A master seed has been generated into /home/martin/.signer/secrets.dat
+
+This is required to be able to store credentials, such as :
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it.
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+```
+
+(for readability purposes, we'll remove the WARNING printout in the rest of this document)
+
+## Creating rules
+
+Now, you can create a rule-file.
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+```
+Get the `sha256` hash....
+```text
+#sha256sum rules.js
+6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
+```
+...And then `attest` the file:
+```text
+#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+
+INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+```
+At this point, we then start the signer with the rule-file:
+
+```text
+#./signer --rules rules.json
+
+INFO [02-21|12:15:18] Using CLI as UI-channel
+INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+DEBUG[02-21|12:15:18] Ledger support enabled
+DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+
+```
+
+Any list-requests will now be auto-approved by our rule-file.
+
+## Under the hood
+
+While doing the operations above, these files have been created:
+
+```text
+#ls -laR ~/.signer/
+/home/martin/.signer/:
+total 16
+drwx------  3 martin martin 4096 feb 21 12:14 .
+drwxr-xr-x 71 martin martin 4096 feb 21 12:12 ..
+drwx------  2 martin martin 4096 feb 21 12:14 43f73718397aa54d1b22
+-rwx------  1 martin martin  256 feb 21 12:12 secrets.dat
+
+/home/martin/.signer/43f73718397aa54d1b22:
+total 12
+drwx------ 2 martin martin 4096 feb 21 12:14 .
+drwx------ 3 martin martin 4096 feb 21 12:14 ..
+-rw------- 1 martin martin  159 feb 21 12:14 config.json
+
+#cat /home/martin/.signer/43f73718397aa54d1b22/config.json
+{"ruleset_sha256":{"iv":"6v4W4tfJxj3zZFbl","c":"6dt5RTDiTq93yh1qDEjpsat/tsKG7cb+vr3sza26IPL2fvsQ6ZoqFx++CPUa8yy6fD9Bbq41L01ehkKHTG3pOAeqTW6zc/+t0wv3AB6xPmU="}}
+
+```
+
+In `~/.signer`, the `secrets.dat` file was created, containing the `master_seed`.
+The `master_seed` was then used to derive a few other things:
+
+- `vault_location` : in this case `43f73718397aa54d1b22` .
+   - Thus, if you use a different `master_seed`, another `vault_location` will be used that does not conflict with each other.
+   - Example: `signer --signersecret /path/to/afile ...`
+- `config.json` which is the encrypted key/value storage for configuration data, containing the key `ruleset_sha256`.
+
+
+## Adding credentials
+
+In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+
+```text
+#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+
+INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
+```
+## More advanced rules
+
+Now let's update the rules to make use of credentials
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+function ApproveSignData(r){
+    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+    {
+        if(r.message.indexOf("bazonk") >= 0){
+            return "Approve"
+        }
+        return "Reject"
+    }
+    // Otherwise goes to manual processing
+}
+
+```
+In this example,
+* any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`,
+    * and auto-rejected if it does not.
+    * Any other signing-requests will be passed along for manual approve/reject.
+
+..attest the new file
+```text
+#sha256sum rules.js
+2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
+
+#./signer attest 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+
+INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+```
+
+And start the signer:
+
+```
+#./signer --rules rules.js
+
+INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[02-21|14:41:56] Trezor support enabled
+INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
+```
+And then test signing, once with `bazonk` and once without:
+
+```
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bazonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"result":"0x93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c"}
+
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"error":{"code":-32000,"message":"Request denied"}}
+
+```
+
+Meanwhile, in the signer output:
+```text
+INFO [02-21|14:42:41] Op approved
+INFO [02-21|14:42:56] Op rejected
+```
+
+The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
+
+```text
+#tail audit.log -n 4
+t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
+```

cmd/ethkey/main.go

@@ -53,10 +53,6 @@ var (
 		Name:  "json",
 		Usage: "output JSON instead of human-readable format",
 	}
-	messageFlag = cli.StringFlag{
-		Name:  "message",
-		Usage: "the file that contains the message to sign/verify",
-	}
 )
 
 func main() {

cmd/evm/runner.go

@@ -76,6 +76,7 @@ func runCmd(ctx *cli.Context) error {
 	logconfig := &vm.LogConfig{
 		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
 		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+		Debug:         ctx.GlobalBool(DebugFlag.Name),
 	}
 
 	var (
@@ -83,8 +84,8 @@ func runCmd(ctx *cli.Context) error {
 		debugLogger *vm.StructLogger
 		statedb     *state.StateDB
 		chainConfig *params.ChainConfig
-		sender      = common.StringToAddress("sender")
-		receiver    = common.StringToAddress("receiver")
+		sender      = common.BytesToAddress([]byte("sender"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -234,9 +235,7 @@ Gas used:           %d
 
 `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
 	}
-	if tracer != nil {
-		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime, err)
-	} else {
+	if tracer == nil {
 		fmt.Printf("0x%x\n", ret)
 		if err != nil {
 			fmt.Printf(" error: %v\n", err)

cmd/geth/bugcmd.go

@@ -49,15 +49,17 @@ func reportBug(ctx *cli.Context) error {
 	// execute template and write contents to buff
 	var buff bytes.Buffer
 
-	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "#### System information")
+	fmt.Fprintln(&buff)
 	fmt.Fprintln(&buff, "Version:", params.Version)
 	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
 	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
 	printOSDetails(&buff)
+	fmt.Fprintln(&buff, header)
 
 	// open a new GH issue
 	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
-		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+		fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String())
 	}
 	return nil
 }
@@ -97,13 +99,15 @@ func printCmdOut(w io.Writer, prefix, path string, args ...string) {
 	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
 }
 
-const header = `Please answer these questions before submitting your issue. Thanks!
+const header = `
+#### Expected behaviour
 
-#### What did you do?
- 
-#### What did you expect to see?
- 
-#### What did you see instead?
- 
-#### System details
+
+#### Actual behaviour
+
+
+#### Steps to reproduce the behaviour
+
+
+#### Backtrace
 `

cmd/geth/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -46,8 +45,6 @@ const (
 var (
 	// Git SHA1 commit hash of the release (set via linker flags)
 	gitCommit = ""
-	// Ethereum address of the Geth release oracle.
-	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
 	// flags that configure the node
@@ -149,7 +146,7 @@ func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2017 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2018 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
@@ -244,7 +241,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	stack.AccountManager().Subscribe(events)
 
 	go func() {
-		// Create an chain state reader for self-derivation
+		// Create a chain state reader for self-derivation
 		rpcClient, err := stack.Attach()
 		if err != nil {
 			utils.Fatalf("Failed to attach to self: %v", err)

cmd/geth/usage.go

@@ -33,7 +33,7 @@ import (
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2017 The go-ethereum Authors
+   Copyright 2013-2018 The go-ethereum Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}

cmd/puppeth/module_dashboard.go

@@ -683,7 +683,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
-// dashboardInfos is returned from an dashboard status check to allow reporting
+// dashboardInfos is returned from a dashboard status check to allow reporting
 // various configuration parameters.
 type dashboardInfos struct {
 	host    string

cmd/puppeth/module_explorer.go

@@ -168,7 +168,7 @@ func (info *explorerInfos) Report() map[string]string {
 	return report
 }
 
-// checkExplorer does a health-check against an block explorer server to verify
+// checkExplorer does a health-check against a block explorer server to verify
 // whether it's running, and if yes, whether it's responsive.
 func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
 	// Inspect a possible block explorer container on the host

cmd/puppeth/module_faucet.go

@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
-// faucetDockerfile is the Dockerfile required to build an faucet container to
+// faucetDockerfile is the Dockerfile required to build a faucet container to
 // grant crypto tokens based on GitHub authentications.
 var faucetDockerfile = `
 FROM ethereum/client-go:alltools-latest
@@ -138,7 +138,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
-// faucetInfos is returned from an faucet status check to allow reporting various
+// faucetInfos is returned from a faucet status check to allow reporting various
 // configuration parameters.
 type faucetInfos struct {
 	node          *nodeInfos
@@ -181,7 +181,7 @@ func (info *faucetInfos) Report() map[string]string {
 	return report
 }
 
-// checkFaucet does a health-check against an faucet server to verify whether
+// checkFaucet does a health-check against a faucet server to verify whether
 // it's running, and if yes, gathering a collection of useful infos about it.
 func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
 	// Inspect a possible faucet container on the host

cmd/puppeth/module_node.go

@@ -198,7 +198,7 @@ func (info *nodeInfos) Report() map[string]string {
 	return report
 }
 
-// checkNode does a health-check against an boot or seal node server to verify
+// checkNode does a health-check against a boot or seal node server to verify
 // whether it's running, and if yes, whether it's responsive.
 func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) {
 	kind := "bootnode"

cmd/utils/flags.go

@@ -64,7 +64,7 @@ var (
 {{if .cmd.Description}}{{.cmd.Description}}
 {{end}}{{if .cmd.Subcommands}}
 SUBCOMMANDS:
-	{{range .cmd.Subcommands}}{{.cmd.Name}}{{with .cmd.ShortName}}, {{.cmd}}{{end}}{{ "\t" }}{{.cmd.Usage}}
+	{{range .cmd.Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
 	{{end}}{{end}}{{if .categorizedFlags}}
 {{range $idx, $categorized := .categorizedFlags}}{{$categorized.Name}} OPTIONS:
 {{range $categorized.Flags}}{{"\t"}}{{.}}

common/types.go

@@ -18,10 +18,12 @@ package common
 
 import (
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"math/big"
 	"math/rand"
 	"reflect"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
@@ -45,9 +47,8 @@ func BytesToHash(b []byte) Hash {
 	h.SetBytes(b)
 	return h
 }
-func StringToHash(s string) Hash { return BytesToHash([]byte(s)) }
-func BigToHash(b *big.Int) Hash  { return BytesToHash(b.Bytes()) }
-func HexToHash(s string) Hash    { return BytesToHash(FromHex(s)) }
+func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) }
+func HexToHash(s string) Hash   { return BytesToHash(FromHex(s)) }
 
 // Get the string representation of the underlying hash
 func (h Hash) Str() string   { return string(h[:]) }
@@ -143,9 +144,8 @@ func BytesToAddress(b []byte) Address {
 	a.SetBytes(b)
 	return a
 }
-func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) }
-func BigToAddress(b *big.Int) Address  { return BytesToAddress(b.Bytes()) }
-func HexToAddress(s string) Address    { return BytesToAddress(FromHex(s)) }
+func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) }
+func HexToAddress(s string) Address   { return BytesToAddress(FromHex(s)) }
 
 // IsHexAddress verifies whether a string can represent a valid hex-encoded
 // Ethereum address or not.
@@ -240,3 +240,63 @@ func (a *UnprefixedAddress) UnmarshalText(input []byte) error {
 func (a UnprefixedAddress) MarshalText() ([]byte, error) {
 	return []byte(hex.EncodeToString(a[:])), nil
 }
+
+// MixedcaseAddress retains the original string, which may or may not be
+// correctly checksummed
+type MixedcaseAddress struct {
+	addr     Address
+	original string
+}
+
+// NewMixedcaseAddress constructor (mainly for testing)
+func NewMixedcaseAddress(addr Address) MixedcaseAddress {
+	return MixedcaseAddress{addr: addr, original: addr.Hex()}
+}
+
+// NewMixedcaseAddressFromString is mainly meant for unit-testing
+func NewMixedcaseAddressFromString(hexaddr string) (*MixedcaseAddress, error) {
+	if !IsHexAddress(hexaddr) {
+		return nil, fmt.Errorf("Invalid address")
+	}
+	a := FromHex(hexaddr)
+	return &MixedcaseAddress{addr: BytesToAddress(a), original: hexaddr}, nil
+}
+
+// UnmarshalJSON parses MixedcaseAddress
+func (ma *MixedcaseAddress) UnmarshalJSON(input []byte) error {
+	if err := hexutil.UnmarshalFixedJSON(addressT, input, ma.addr[:]); err != nil {
+		return err
+	}
+	return json.Unmarshal(input, &ma.original)
+}
+
+// MarshalJSON marshals the original value
+func (ma *MixedcaseAddress) MarshalJSON() ([]byte, error) {
+	if strings.HasPrefix(ma.original, "0x") || strings.HasPrefix(ma.original, "0X") {
+		return json.Marshal(fmt.Sprintf("0x%s", ma.original[2:]))
+	}
+	return json.Marshal(fmt.Sprintf("0x%s", ma.original))
+}
+
+// Address returns the address
+func (ma *MixedcaseAddress) Address() Address {
+	return ma.addr
+}
+
+// String implements fmt.Stringer
+func (ma *MixedcaseAddress) String() string {
+	if ma.ValidChecksum() {
+		return fmt.Sprintf("%s [chksum ok]", ma.original)
+	}
+	return fmt.Sprintf("%s [chksum INVALID]", ma.original)
+}
+
+// ValidChecksum returns true if the address has valid checksum
+func (ma *MixedcaseAddress) ValidChecksum() bool {
+	return ma.original == ma.addr.Hex()
+}
+
+// Original returns the mixed-case input string
+func (ma *MixedcaseAddress) Original() string {
+	return ma.original
+}

common/types_test.go

@@ -18,6 +18,7 @@ package common
 
 import (
 	"encoding/json"
+
 	"math/big"
 	"strings"
 	"testing"
@@ -149,3 +150,46 @@ func BenchmarkAddressHex(b *testing.B) {
 		testAddr.Hex()
 	}
 }
+
+func TestMixedcaseAccount_Address(t *testing.T) {
+
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md
+	// Note: 0X{checksum_addr} is not valid according to spec above
+
+	var res []struct {
+		A     MixedcaseAddress
+		Valid bool
+	}
+	if err := json.Unmarshal([]byte(`[
+		{"A" : "0xae967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0xAe967917c465db8578ca9024c205720b1a3651A9", "Valid": true},
+		{"A" : "0XAe967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0x1111111111111111111112222222222223333323", "Valid": true}
+		]`), &res); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, r := range res {
+		if got := r.A.ValidChecksum(); got != r.Valid {
+			t.Errorf("Expected checksum %v, got checksum %v, input %v", r.Valid, got, r.A.String())
+		}
+	}
+
+	//These should throw exceptions:
+	var r2 []MixedcaseAddress
+	for _, r := range []string{
+		`["0x11111111111111111111122222222222233333"]`,     // Too short
+		`["0x111111111111111111111222222222222333332"]`,    // Too short
+		`["0x11111111111111111111122222222222233333234"]`,  // Too long
+		`["0x111111111111111111111222222222222333332344"]`, // Too long
+		`["1111111111111111111112222222222223333323"]`,     // Missing 0x
+		`["x1111111111111111111112222222222223333323"]`,    // Missing 0
+		`["0xG111111111111111111112222222222223333323"]`,   //Non-hex
+	} {
+		if err := json.Unmarshal([]byte(r), &r2); err == nil {
+			t.Errorf("Expected failure, input %v", r)
+		}
+
+	}
+
+}

compression/rle/read_write.go

@@ -1,101 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package rle implements the run-length encoding used for Ethereum data.
-package rle
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-const (
-	token             byte = 0xfe
-	emptyShaToken          = 0xfd
-	emptyListShaToken      = 0xfe
-	tokenToken             = 0xff
-)
-
-var empty = crypto.Keccak256([]byte(""))
-var emptyList = crypto.Keccak256([]byte{0x80})
-
-func Decompress(dat []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-
-	for i := 0; i < len(dat); i++ {
-		if dat[i] == token {
-			if i+1 < len(dat) {
-				switch dat[i+1] {
-				case emptyShaToken:
-					buf.Write(empty)
-				case emptyListShaToken:
-					buf.Write(emptyList)
-				case tokenToken:
-					buf.WriteByte(token)
-				default:
-					buf.Write(make([]byte, int(dat[i+1]-2)))
-				}
-				i++
-			} else {
-				return nil, errors.New("error reading bytes. token encountered without proceeding bytes")
-			}
-		} else {
-			buf.WriteByte(dat[i])
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func compressChunk(dat []byte) (ret []byte, n int) {
-	switch {
-	case dat[0] == token:
-		return []byte{token, tokenToken}, 1
-	case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0:
-		j := 0
-		for j <= 254 && j < len(dat) {
-			if dat[j] != 0 {
-				break
-			}
-			j++
-		}
-		return []byte{token, byte(j + 2)}, j
-	case len(dat) >= 32:
-		if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) {
-			return []byte{token, emptyShaToken}, 32
-		} else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) {
-			return []byte{token, emptyListShaToken}, 32
-		}
-		fallthrough
-	default:
-		return dat[:1], 1
-	}
-}
-
-func Compress(dat []byte) []byte {
-	buf := new(bytes.Buffer)
-
-	i := 0
-	for i < len(dat) {
-		b, n := compressChunk(dat[i:])
-		buf.Write(b)
-		i += n
-	}
-
-	return buf.Bytes()
-}

compression/rle/read_write_test.go

@@ -1,50 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package rle
-
-import (
-	"testing"
-
-	checker "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { checker.TestingT(t) }
-
-type CompressionRleSuite struct{}
-
-var _ = checker.Suite(&CompressionRleSuite{})
-
-func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) {
-	exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70}
-	res, err := Decompress([]byte{token, 0xfd})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21}
-	res, err = Decompress([]byte{token, 0xfe})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	res, err = Decompress([]byte{token, 0xff})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, []byte{token})
-
-	res, err = Decompress([]byte{token, 12})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, make([]byte, 10))
-
-}

consensus/consensus.go

@@ -18,12 +18,13 @@
 package consensus
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"math/big"
 )
 
 // ChainReader defines a small collection of methods needed to access the local

console/bridge.go

@@ -87,7 +87,7 @@ func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) {
 // OpenWallet is a wrapper around personal.openWallet which can interpret and
 // react to certain error messages, such as the Trezor PIN matrix request.
 func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) {
-	// Make sure we have an wallet specified to open
+	// Make sure we have a wallet specified to open
 	if !call.Argument(0).IsString() {
 		throwJSException("first argument must be the wallet URL to open")
 	}

core/asm/compiler.go

@@ -17,7 +17,6 @@
 package asm
 
 import (
-	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -237,19 +236,16 @@ func (c *Compiler) pushBin(v interface{}) {
 // isPush returns whether the string op is either any of
 // push(N).
 func isPush(op string) bool {
-	return op == "push"
+	return strings.ToUpper(op) == "PUSH"
 }
 
 // isJump returns whether the string op is jump(i)
 func isJump(op string) bool {
-	return op == "jumpi" || op == "jump"
+	return strings.ToUpper(op) == "JUMPI" || strings.ToUpper(op) == "JUMP"
 }
 
 // toBinary converts text to a vm.OpCode
 func toBinary(text string) vm.OpCode {
-	if isPush(text) {
-		text = "push1"
-	}
 	return vm.StringToOp(strings.ToUpper(text))
 }
 
@@ -264,11 +260,6 @@ func (err compileError) Error() string {
 	return fmt.Sprintf("%d syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
 }
 
-var (
-	errExpBol            = errors.New("expected beginning of line")
-	errExpElementOrLabel = errors.New("expected beginning of line")
-)
-
 func compileErr(c token, got, want string) error {
 	return compileError{
 		got:    got,

core/blockchain_test.go

@@ -1338,3 +1338,114 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		}
 	}
 }
+
+// Benchmarks large blocks with value transfers to non-existing accounts
+func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) {
+	var (
+		signer          = types.HomesteadSigner{}
+		testBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+		bankFunds       = big.NewInt(100000000000000000)
+		gspec           = Genesis{
+			Config: params.TestChainConfig,
+			Alloc: GenesisAlloc{
+				testBankAddress: {Balance: bankFunds},
+				common.HexToAddress("0xc0de"): {
+					Code:    []byte{0x60, 0x01, 0x50},
+					Balance: big.NewInt(0),
+				}, // push 1, pop
+			},
+			GasLimit: 100e6, // 100 M
+		}
+	)
+	// Generate the original common chain segment and the two competing forks
+	engine := ethash.NewFaker()
+	db, _ := ethdb.NewMemDatabase()
+	genesis := gspec.MustCommit(db)
+
+	blockGenerator := func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{1})
+		for txi := 0; txi < numTxs; txi++ {
+			uniq := uint64(i*numTxs + txi)
+			recipient := recipientFn(uniq)
+			//recipient := common.BigToAddress(big.NewInt(0).SetUint64(1337 + uniq))
+			tx, err := types.SignTx(types.NewTransaction(uniq, recipient, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testBankKey)
+			if err != nil {
+				b.Error(err)
+			}
+			block.AddTx(tx)
+		}
+	}
+
+	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, numBlocks, blockGenerator)
+	b.StopTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Import the shared chain and the original canonical one
+		diskdb, _ := ethdb.NewMemDatabase()
+		gspec.MustCommit(diskdb)
+
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+		if err != nil {
+			b.Fatalf("failed to create tester chain: %v", err)
+		}
+		b.StartTimer()
+		if _, err := chain.InsertChain(shared); err != nil {
+			b.Fatalf("failed to insert shared chain: %v", err)
+		}
+		b.StopTimer()
+		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
+			b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
+
+		}
+	}
+}
+func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}

core/bloombits/matcher.go

@@ -392,7 +392,7 @@ func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
 		shutdown = session.quit // Shutdown request channel, will gracefully wait for pending requests
 	)
 
-	// assign is a helper method fo try to assign a pending bit an an actively
+	// assign is a helper method fo try to assign a pending bit an actively
 	// listening servicer, or schedule it up for later when one arrives.
 	assign := func(bit uint) {
 		select {

core/database_util_test.go

@@ -317,7 +317,7 @@ func TestLookupStorage(t *testing.T) {
 			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
 				t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
 			}
-			if tx.String() != txn.String() {
+			if tx.Hash() != txn.Hash() {
 				t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
 			}
 		}

core/state/database.go

@@ -26,7 +26,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-// Trie cache generation limit after which to evic trie nodes from memory.
+// Trie cache generation limit after which to evict trie nodes from memory.
 var MaxTrieCacheGen = uint16(120)
 
 const (
@@ -151,9 +151,6 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 		return cached.(int), nil
 	}
 	code, err := db.ContractCode(addrHash, codeHash)
-	if err == nil {
-		db.codeSizeCache.Add(codeHash, len(code))
-	}
 	return len(code), err
 }
 

core/state/dump.go

@@ -53,7 +53,7 @@ func (self *StateDB) RawDump() Dump {
 			panic(err)
 		}
 
-		obj := newObject(nil, common.BytesToAddress(addr), data, nil)
+		obj := newObject(nil, common.BytesToAddress(addr), data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,

core/state/journal.go

@@ -22,11 +22,67 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// journalEntry is a modification entry in the state change journal that can be
+// reverted on demand.
 type journalEntry interface {
-	undo(*StateDB)
+	// revert undoes the changes introduced by this journal entry.
+	revert(*StateDB)
+
+	// dirtied returns the Ethereum address modified by this journal entry.
+	dirtied() *common.Address
 }
 
-type journal []journalEntry
+// journal contains the list of state modifications applied since the last state
+// commit. These are tracked to be able to be reverted in case of an execution
+// exception or revertal request.
+type journal struct {
+	entries []journalEntry         // Current changes tracked by the journal
+	dirties map[common.Address]int // Dirty accounts and the number of changes
+}
+
+// newJournal create a new initialized journal.
+func newJournal() *journal {
+	return &journal{
+		dirties: make(map[common.Address]int),
+	}
+}
+
+// append inserts a new modification entry to the end of the change journal.
+func (j *journal) append(entry journalEntry) {
+	j.entries = append(j.entries, entry)
+	if addr := entry.dirtied(); addr != nil {
+		j.dirties[*addr]++
+	}
+}
+
+// revert undoes a batch of journalled modifications along with any reverted
+// dirty handling too.
+func (j *journal) revert(statedb *StateDB, snapshot int) {
+	for i := len(j.entries) - 1; i >= snapshot; i-- {
+		// Undo the changes made by the operation
+		j.entries[i].revert(statedb)
+
+		// Drop any dirty tracking induced by the change
+		if addr := j.entries[i].dirtied(); addr != nil {
+			if j.dirties[*addr]--; j.dirties[*addr] == 0 {
+				delete(j.dirties, *addr)
+			}
+		}
+	}
+	j.entries = j.entries[:snapshot]
+}
+
+// dirty explicitly sets an address to dirty, even if the change entries would
+// otherwise suggest it as clean. This method is an ugly hack to handle the RIPEMD
+// precompile consensus exception.
+func (j *journal) dirty(addr common.Address) {
+	j.dirties[addr]++
+}
+
+// length returns the current number of entries in the journal.
+func (j *journal) length() int {
+	return len(j.entries)
+}
 
 type (
 	// Changes to the account trie.
@@ -77,16 +133,24 @@ type (
 	}
 )
 
-func (ch createObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) revert(s *StateDB) {
 	delete(s.stateObjects, *ch.account)
 	delete(s.stateObjectsDirty, *ch.account)
 }
 
-func (ch resetObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch resetObjectChange) revert(s *StateDB) {
 	s.setStateObject(ch.prev)
 }
 
-func (ch suicideChange) undo(s *StateDB) {
+func (ch resetObjectChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch suicideChange) revert(s *StateDB) {
 	obj := s.getStateObject(*ch.account)
 	if obj != nil {
 		obj.suicided = ch.prev
@@ -94,38 +158,60 @@ func (ch suicideChange) undo(s *StateDB) {
 	}
 }
 
-var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
-
-func (ch touchChange) undo(s *StateDB) {
-	if !ch.prev && *ch.account != ripemd {
-		s.getStateObject(*ch.account).touched = ch.prev
-		if !ch.prevDirty {
-			delete(s.stateObjectsDirty, *ch.account)
-		}
-	}
+func (ch suicideChange) dirtied() *common.Address {
+	return ch.account
 }
 
-func (ch balanceChange) undo(s *StateDB) {
+var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
+
+func (ch touchChange) revert(s *StateDB) {
+}
+
+func (ch touchChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch balanceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setBalance(ch.prev)
 }
 
-func (ch nonceChange) undo(s *StateDB) {
+func (ch balanceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch nonceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setNonce(ch.prev)
 }
 
-func (ch codeChange) undo(s *StateDB) {
+func (ch nonceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch codeChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
 }
 
-func (ch storageChange) undo(s *StateDB) {
+func (ch codeChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch storageChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setState(ch.key, ch.prevalue)
 }
 
-func (ch refundChange) undo(s *StateDB) {
+func (ch storageChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch refundChange) revert(s *StateDB) {
 	s.refund = ch.prev
 }
 
-func (ch addLogChange) undo(s *StateDB) {
+func (ch refundChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addLogChange) revert(s *StateDB) {
 	logs := s.logs[ch.txhash]
 	if len(logs) == 1 {
 		delete(s.logs, ch.txhash)
@@ -135,6 +221,14 @@ func (ch addLogChange) undo(s *StateDB) {
 	s.logSize--
 }
 
-func (ch addPreimageChange) undo(s *StateDB) {
+func (ch addLogChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addPreimageChange) revert(s *StateDB) {
 	delete(s.preimages, ch.hash)
 }
+
+func (ch addPreimageChange) dirtied() *common.Address {
+	return nil
+}

core/state/state_object.go

@@ -85,9 +85,7 @@ type stateObject struct {
 	// during the "update" phase of the state transition.
 	dirtyCode bool // true if the code was updated
 	suicided  bool
-	touched   bool
 	deleted   bool
-	onDirty   func(addr common.Address) // Callback method to mark a state object newly dirty
 }
 
 // empty returns whether the account is considered empty.
@@ -105,7 +103,7 @@ type Account struct {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, data Account, onDirty func(addr common.Address)) *stateObject {
+func newObject(db *StateDB, address common.Address, data Account) *stateObject {
 	if data.Balance == nil {
 		data.Balance = new(big.Int)
 	}
@@ -119,7 +117,6 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a
 		data:          data,
 		cachedStorage: make(Storage),
 		dirtyStorage:  make(Storage),
-		onDirty:       onDirty,
 	}
 }
 
@@ -137,23 +134,17 @@ func (self *stateObject) setError(err error) {
 
 func (self *stateObject) markSuicided() {
 	self.suicided = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (c *stateObject) touch() {
-	c.db.journal = append(c.db.journal, touchChange{
-		account:   &c.address,
-		prev:      c.touched,
-		prevDirty: c.onDirty == nil,
+	c.db.journal.append(touchChange{
+		account: &c.address,
 	})
-	if c.onDirty != nil {
-		c.onDirty(c.Address())
-		c.onDirty = nil
+	if c.address == ripemd {
+		// Explicitly put it in the dirty-cache, which is otherwise generated from
+		// flattened journals.
+		c.db.journal.dirty(c.address)
 	}
-	c.touched = true
 }
 
 func (c *stateObject) getTrie(db Database) Trie {
@@ -195,7 +186,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash {
 
 // SetState updates a value in account storage.
 func (self *stateObject) SetState(db Database, key, value common.Hash) {
-	self.db.journal = append(self.db.journal, storageChange{
+	self.db.journal.append(storageChange{
 		account:  &self.address,
 		key:      key,
 		prevalue: self.GetState(db, key),
@@ -207,10 +198,6 @@ func (self *stateObject) setState(key, value common.Hash) {
 	self.cachedStorage[key] = value
 	self.dirtyStorage[key] = value
 
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // updateTrie writes cached storage modifications into the object's storage trie.
@@ -274,7 +261,7 @@ func (c *stateObject) SubBalance(amount *big.Int) {
 }
 
 func (self *stateObject) SetBalance(amount *big.Int) {
-	self.db.journal = append(self.db.journal, balanceChange{
+	self.db.journal.append(balanceChange{
 		account: &self.address,
 		prev:    new(big.Int).Set(self.data.Balance),
 	})
@@ -283,17 +270,13 @@ func (self *stateObject) SetBalance(amount *big.Int) {
 
 func (self *stateObject) setBalance(amount *big.Int) {
 	self.data.Balance = amount
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // Return the gas back to the origin. Used by the Virtual machine or Closures
 func (c *stateObject) ReturnGas(gas *big.Int) {}
 
-func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject {
-	stateObject := newObject(db, self.address, self.data, onDirty)
+func (self *stateObject) deepCopy(db *StateDB) *stateObject {
+	stateObject := newObject(db, self.address, self.data)
 	if self.trie != nil {
 		stateObject.trie = db.db.CopyTrie(self.trie)
 	}
@@ -333,7 +316,7 @@ func (self *stateObject) Code(db Database) []byte {
 
 func (self *stateObject) SetCode(codeHash common.Hash, code []byte) {
 	prevcode := self.Code(self.db.db)
-	self.db.journal = append(self.db.journal, codeChange{
+	self.db.journal.append(codeChange{
 		account:  &self.address,
 		prevhash: self.CodeHash(),
 		prevcode: prevcode,
@@ -345,14 +328,10 @@ func (self *stateObject) setCode(codeHash common.Hash, code []byte) {
 	self.code = code
 	self.data.CodeHash = codeHash[:]
 	self.dirtyCode = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) SetNonce(nonce uint64) {
-	self.db.journal = append(self.db.journal, nonceChange{
+	self.db.journal.append(nonceChange{
 		account: &self.address,
 		prev:    self.data.Nonce,
 	})
@@ -361,10 +340,6 @@ func (self *stateObject) SetNonce(nonce uint64) {
 
 func (self *stateObject) setNonce(nonce uint64) {
 	self.data.Nonce = nonce
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) CodeHash() []byte {

core/state/statedb.go

@@ -76,7 +76,7 @@ type StateDB struct {
 
 	// Journal of state modifications. This is the backbone of
 	// Snapshot and RevertToSnapshot.
-	journal        journal
+	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
 
@@ -96,6 +96,7 @@ func New(root common.Hash, db Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		logs:              make(map[common.Hash][]*types.Log),
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}, nil
 }
 
@@ -131,7 +132,7 @@ func (self *StateDB) Reset(root common.Hash) error {
 }
 
 func (self *StateDB) AddLog(log *types.Log) {
-	self.journal = append(self.journal, addLogChange{txhash: self.thash})
+	self.journal.append(addLogChange{txhash: self.thash})
 
 	log.TxHash = self.thash
 	log.BlockHash = self.bhash
@@ -156,7 +157,7 @@ func (self *StateDB) Logs() []*types.Log {
 // AddPreimage records a SHA3 preimage seen by the VM.
 func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) {
 	if _, ok := self.preimages[hash]; !ok {
-		self.journal = append(self.journal, addPreimageChange{hash: hash})
+		self.journal.append(addPreimageChange{hash: hash})
 		pi := make([]byte, len(preimage))
 		copy(pi, preimage)
 		self.preimages[hash] = pi
@@ -169,7 +170,7 @@ func (self *StateDB) Preimages() map[common.Hash][]byte {
 }
 
 func (self *StateDB) AddRefund(gas uint64) {
-	self.journal = append(self.journal, refundChange{prev: self.refund})
+	self.journal.append(refundChange{prev: self.refund})
 	self.refund += gas
 }
 
@@ -235,10 +236,10 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash {
 	return common.BytesToHash(stateObject.CodeHash())
 }
 
-func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) GetState(addr common.Address, bhash common.Hash) common.Hash {
+	stateObject := self.getStateObject(addr)
 	if stateObject != nil {
-		return stateObject.GetState(self.db, b)
+		return stateObject.GetState(self.db, bhash)
 	}
 	return common.Hash{}
 }
@@ -250,12 +251,12 @@ func (self *StateDB) Database() Database {
 
 // StorageTrie returns the storage trie of an account.
 // The return value is a copy and is nil for non-existent accounts.
-func (self *StateDB) StorageTrie(a common.Address) Trie {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) StorageTrie(addr common.Address) Trie {
+	stateObject := self.getStateObject(addr)
 	if stateObject == nil {
 		return nil
 	}
-	cpy := stateObject.deepCopy(self, nil)
+	cpy := stateObject.deepCopy(self)
 	return cpy.updateTrie(self.db)
 }
 
@@ -271,7 +272,7 @@ func (self *StateDB) HasSuicided(addr common.Address) bool {
  * SETTERS
  */
 
-// AddBalance adds amount to the account associated with addr
+// AddBalance adds amount to the account associated with addr.
 func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -279,7 +280,7 @@ func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	}
 }
 
-// SubBalance subtracts amount from the account associated with addr
+// SubBalance subtracts amount from the account associated with addr.
 func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -308,7 +309,7 @@ func (self *StateDB) SetCode(addr common.Address, code []byte) {
 	}
 }
 
-func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
+func (self *StateDB) SetState(addr common.Address, key, value common.Hash) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
 		stateObject.SetState(self.db, key, value)
@@ -325,7 +326,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 	if stateObject == nil {
 		return false
 	}
-	self.journal = append(self.journal, suicideChange{
+	self.journal.append(suicideChange{
 		account:     &addr,
 		prev:        stateObject.suicided,
 		prevbalance: new(big.Int).Set(stateObject.Balance()),
@@ -337,7 +338,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 }
 
 //
-// Setting, updating & deleting state object methods
+// Setting, updating & deleting state object methods.
 //
 
 // updateStateObject writes the given object to the trie.
@@ -379,7 +380,7 @@ func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObje
 		return nil
 	}
 	// Insert into the live set.
-	obj := newObject(self, addr, data, self.MarkStateObjectDirty)
+	obj := newObject(self, addr, data)
 	self.setStateObject(obj)
 	return obj
 }
@@ -388,7 +389,7 @@ func (self *StateDB) setStateObject(object *stateObject) {
 	self.stateObjects[object.Address()] = object
 }
 
-// Retrieve a state object or create a new state object if nil
+// Retrieve a state object or create a new state object if nil.
 func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	stateObject := self.getStateObject(addr)
 	if stateObject == nil || stateObject.deleted {
@@ -397,22 +398,16 @@ func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	return stateObject
 }
 
-// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly
-// state object cache iteration to find a handful of modified ones.
-func (self *StateDB) MarkStateObjectDirty(addr common.Address) {
-	self.stateObjectsDirty[addr] = struct{}{}
-}
-
 // createObject creates a new state object. If there is an existing account with
 // the given address, it is overwritten and returned as the second return value.
 func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = self.getStateObject(addr)
-	newobj = newObject(self, addr, Account{}, self.MarkStateObjectDirty)
+	newobj = newObject(self, addr, Account{})
 	newobj.setNonce(0) // sets the object to dirty
 	if prev == nil {
-		self.journal = append(self.journal, createObjectChange{account: &addr})
+		self.journal.append(createObjectChange{account: &addr})
 	} else {
-		self.journal = append(self.journal, resetObjectChange{prev: prev})
+		self.journal.append(resetObjectChange{prev: prev})
 	}
 	self.setStateObject(newobj)
 	return newobj, prev
@@ -466,18 +461,35 @@ func (self *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                self.db,
 		trie:              self.db.CopyTrie(self.trie),
-		stateObjects:      make(map[common.Address]*stateObject, len(self.stateObjectsDirty)),
-		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
+		stateObjects:      make(map[common.Address]*stateObject, len(self.journal.dirties)),
+		stateObjectsDirty: make(map[common.Address]struct{}, len(self.journal.dirties)),
 		refund:            self.refund,
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}
 	// Copy the dirty states, logs, and preimages
-	for addr := range self.stateObjectsDirty {
-		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty)
-		state.stateObjectsDirty[addr] = struct{}{}
+	for addr := range self.journal.dirties {
+		// As documented [here](https://github.com/ethereum/go-ethereum/pull/16485#issuecomment-380438527),
+		// and in the Finalise-method, there is a case where an object is in the journal but not
+		// in the stateObjects: OOG after touch on ripeMD prior to Byzantium. Thus, we need to check for
+		// nil
+		if object, exist := self.stateObjects[addr]; exist {
+			state.stateObjects[addr] = object.deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
 	}
+	// Above, we don't copy the actual journal. This means that if the copy is copied, the
+	// loop above will be a no-op, since the copy's journal is empty.
+	// Thus, here we iterate over stateObjects, to enable copies of copies
+	for addr := range self.stateObjectsDirty {
+		if _, exist := state.stateObjects[addr]; !exist {
+			state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
+	}
+
 	for hash, logs := range self.logs {
 		state.logs[hash] = make([]*types.Log, len(logs))
 		copy(state.logs[hash], logs)
@@ -492,7 +504,7 @@ func (self *StateDB) Copy() *StateDB {
 func (self *StateDB) Snapshot() int {
 	id := self.nextRevisionId
 	self.nextRevisionId++
-	self.validRevisions = append(self.validRevisions, revision{id, len(self.journal)})
+	self.validRevisions = append(self.validRevisions, revision{id, self.journal.length()})
 	return id
 }
 
@@ -507,13 +519,8 @@ func (self *StateDB) RevertToSnapshot(revid int) {
 	}
 	snapshot := self.validRevisions[idx].journalIndex
 
-	// Replay the journal to undo changes.
-	for i := len(self.journal) - 1; i >= snapshot; i-- {
-		self.journal[i].undo(self)
-	}
-	self.journal = self.journal[:snapshot]
-
-	// Remove invalidated snapshots from the stack.
+	// Replay the journal to undo changes and remove invalidated snapshots
+	self.journal.revert(self, snapshot)
 	self.validRevisions = self.validRevisions[:idx]
 }
 
@@ -525,14 +532,25 @@ func (self *StateDB) GetRefund() uint64 {
 // Finalise finalises the state by removing the self destructed objects
 // and clears the journal as well as the refunds.
 func (s *StateDB) Finalise(deleteEmptyObjects bool) {
-	for addr := range s.stateObjectsDirty {
-		stateObject := s.stateObjects[addr]
+	for addr := range s.journal.dirties {
+		stateObject, exist := s.stateObjects[addr]
+		if !exist {
+			// ripeMD is 'touched' at block 1714175, in tx 0x1237f737031e40bcde4a8b7e717b2d15e3ecadfe49bb1bbc71ee9deb09c6fcf2
+			// That tx goes out of gas, and although the notion of 'touched' does not exist there, the
+			// touch-event will still be recorded in the journal. Since ripeMD is a special snowflake,
+			// it will persist in the journal even though the journal is reverted. In this special circumstance,
+			// it may exist in `s.journal.dirties` but not in `s.stateObjects`.
+			// Thus, we can safely ignore it here
+			continue
+		}
+
 		if stateObject.suicided || (deleteEmptyObjects && stateObject.empty()) {
 			s.deleteStateObject(stateObject)
 		} else {
 			stateObject.updateRoot(s.db)
 			s.updateStateObject(stateObject)
 		}
+		s.stateObjectsDirty[addr] = struct{}{}
 	}
 	// Invalidate journal because reverting across transactions is not allowed.
 	s.clearJournalAndRefund()
@@ -576,7 +594,7 @@ func (s *StateDB) DeleteSuicides() {
 }
 
 func (s *StateDB) clearJournalAndRefund() {
-	s.journal = nil
+	s.journal = newJournal()
 	s.validRevisions = s.validRevisions[:0]
 	s.refund = 0
 }
@@ -585,6 +603,9 @@ func (s *StateDB) clearJournalAndRefund() {
 func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
+	for addr := range s.journal.dirties {
+		s.stateObjectsDirty[addr] = struct{}{}
+	}
 	// Commit objects to the trie.
 	for addr, stateObject := range s.stateObjects {
 		_, isDirty := s.stateObjectsDirty[addr]

core/state/statedb_test.go

@@ -413,11 +413,28 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 
 	snapshot := s.state.Snapshot()
 	s.state.AddBalance(common.Address{}, new(big.Int))
-	if len(s.state.stateObjectsDirty) != 1 {
+
+	if len(s.state.journal.dirties) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
 	s.state.RevertToSnapshot(snapshot)
-	if len(s.state.stateObjectsDirty) != 0 {
+	if len(s.state.journal.dirties) != 0 {
 		c.Fatal("expected no dirty state object")
 	}
 }
+
+// TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy.
+// See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512
+func TestCopyOfCopy(t *testing.T) {
+	db, _ := ethdb.NewMemDatabase()
+	sdb, _ := New(common.Hash{}, NewDatabase(db))
+	addr := common.HexToAddress("aaaa")
+	sdb.SetBalance(addr, big.NewInt(42))
+
+	if got := sdb.Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("1st copy fail, expected 42, got %v", got)
+	}
+	if got := sdb.Copy().Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("2nd copy fail, expected 42, got %v", got)
+	}
+}

core/state_transition.go

@@ -132,28 +132,12 @@ func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, uint64, bool,
 	return NewStateTransition(evm, msg, gp).TransitionDb()
 }
 
-func (st *StateTransition) from() vm.AccountRef {
-	f := st.msg.From()
-	if !st.state.Exist(f) {
-		st.state.CreateAccount(f)
+// to returns the recipient of the message.
+func (st *StateTransition) to() common.Address {
+	if st.msg == nil || st.msg.To() == nil /* contract creation */ {
+		return common.Address{}
 	}
-	return vm.AccountRef(f)
-}
-
-func (st *StateTransition) to() vm.AccountRef {
-	if st.msg == nil {
-		return vm.AccountRef{}
-	}
-	to := st.msg.To()
-	if to == nil {
-		return vm.AccountRef{} // contract creation
-	}
-
-	reference := vm.AccountRef(*to)
-	if !st.state.Exist(*to) {
-		st.state.CreateAccount(*to)
-	}
-	return reference
+	return *st.msg.To()
 }
 
 func (st *StateTransition) useGas(amount uint64) error {
@@ -166,12 +150,8 @@ func (st *StateTransition) useGas(amount uint64) error {
 }
 
 func (st *StateTransition) buyGas() error {
-	var (
-		state  = st.state
-		sender = st.from()
-	)
 	mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.Gas()), st.gasPrice)
-	if state.GetBalance(sender.Address()).Cmp(mgval) < 0 {
+	if st.state.GetBalance(st.msg.From()).Cmp(mgval) < 0 {
 		return errInsufficientBalanceForGas
 	}
 	if err := st.gp.SubGas(st.msg.Gas()); err != nil {
@@ -180,20 +160,17 @@ func (st *StateTransition) buyGas() error {
 	st.gas += st.msg.Gas()
 
 	st.initialGas = st.msg.Gas()
-	state.SubBalance(sender.Address(), mgval)
+	st.state.SubBalance(st.msg.From(), mgval)
 	return nil
 }
 
 func (st *StateTransition) preCheck() error {
-	msg := st.msg
-	sender := st.from()
-
-	// Make sure this transaction's nonce is correct
-	if msg.CheckNonce() {
-		nonce := st.state.GetNonce(sender.Address())
-		if nonce < msg.Nonce() {
+	// Make sure this transaction's nonce is correct.
+	if st.msg.CheckNonce() {
+		nonce := st.state.GetNonce(st.msg.From())
+		if nonce < st.msg.Nonce() {
 			return ErrNonceTooHigh
-		} else if nonce > msg.Nonce() {
+		} else if nonce > st.msg.Nonce() {
 			return ErrNonceTooLow
 		}
 	}
@@ -208,8 +185,7 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		return
 	}
 	msg := st.msg
-	sender := st.from() // err checked in preCheck
-
+	sender := vm.AccountRef(msg.From())
 	homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber)
 	contractCreation := msg.To() == nil
 
@@ -233,8 +209,8 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		ret, _, st.gas, vmerr = evm.Create(sender, st.data, st.gas, st.value)
 	} else {
 		// Increment the nonce for the next transaction
-		st.state.SetNonce(sender.Address(), st.state.GetNonce(sender.Address())+1)
-		ret, st.gas, vmerr = evm.Call(sender, st.to().Address(), st.data, st.gas, st.value)
+		st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1)
+		ret, st.gas, vmerr = evm.Call(sender, st.to(), st.data, st.gas, st.value)
 	}
 	if vmerr != nil {
 		log.Debug("VM returned with error", "err", vmerr)
@@ -260,10 +236,8 @@ func (st *StateTransition) refundGas() {
 	st.gas += refund
 
 	// Return ETH for remaining gas, exchanged at the original rate.
-	sender := st.from()
-
 	remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gas), st.gasPrice)
-	st.state.AddBalance(sender.Address(), remaining)
+	st.state.AddBalance(st.msg.From(), remaining)
 
 	// Also return remaining gas to the block gas counter so it is
 	// available for the next transaction.

core/tx_list.go

@@ -367,9 +367,20 @@ func (l *txList) Flatten() types.Transactions {
 // price-sorted transactions to discard when the pool fills up.
 type priceHeap []*types.Transaction
 
-func (h priceHeap) Len() int           { return len(h) }
-func (h priceHeap) Less(i, j int) bool { return h[i].GasPrice().Cmp(h[j].GasPrice()) < 0 }
-func (h priceHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h priceHeap) Len() int      { return len(h) }
+func (h priceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+
+func (h priceHeap) Less(i, j int) bool {
+	// Sort primarily by price, returning the cheaper one
+	switch h[i].GasPrice().Cmp(h[j].GasPrice()) {
+	case -1:
+		return true
+	case 1:
+		return false
+	}
+	// If the prices match, stabilize via nonces (high nonce is worse)
+	return h[i].Nonce() > h[j].Nonce()
+}
 
 func (h *priceHeap) Push(x interface{}) {
 	*h = append(*h, x.(*types.Transaction))

core/tx_pool.go

@@ -38,8 +38,6 @@ import (
 const (
 	// chainHeadChanSize is the size of channel listening to ChainHeadEvent.
 	chainHeadChanSize = 10
-	// rmTxChanSize is the size of channel listening to RemovedTransactionEvent.
-	rmTxChanSize = 10
 )
 
 var (
@@ -320,7 +318,7 @@ func (pool *TxPool) loop() {
 				// Any non-locals old enough should be removed
 				if time.Since(pool.beats[addr]) > pool.config.Lifetime {
 					for _, tx := range pool.queue[addr].Flatten() {
-						pool.removeTx(tx.Hash())
+						pool.removeTx(tx.Hash(), true)
 					}
 				}
 			}
@@ -468,7 +466,7 @@ func (pool *TxPool) SetGasPrice(price *big.Int) {
 
 	pool.gasPrice = price
 	for _, tx := range pool.priced.Cap(price, pool.locals) {
-		pool.removeTx(tx.Hash())
+		pool.removeTx(tx.Hash(), false)
 	}
 	log.Info("Transaction pool price threshold updated", "price", price)
 }
@@ -630,7 +628,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		for _, tx := range drop {
 			log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice())
 			underpricedTxCounter.Inc(1)
-			pool.removeTx(tx.Hash())
+			pool.removeTx(tx.Hash(), false)
 		}
 	}
 	// If the transaction is replacing an already pending one, do directly
@@ -695,8 +693,10 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er
 		pool.priced.Removed()
 		queuedReplaceCounter.Inc(1)
 	}
-	pool.all[hash] = tx
-	pool.priced.Put(tx)
+	if pool.all[hash] == nil {
+		pool.all[hash] = tx
+		pool.priced.Put(tx)
+	}
 	return old != nil, nil
 }
 
@@ -862,7 +862,7 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction {
 
 // removeTx removes a single transaction from the queue, moving all subsequent
 // transactions back to the future queue.
-func (pool *TxPool) removeTx(hash common.Hash) {
+func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 	// Fetch the transaction we wish to delete
 	tx, ok := pool.all[hash]
 	if !ok {
@@ -872,8 +872,9 @@ func (pool *TxPool) removeTx(hash common.Hash) {
 
 	// Remove it from the list of known transactions
 	delete(pool.all, hash)
-	pool.priced.Removed()
-
+	if outofbound {
+		pool.priced.Removed()
+	}
 	// Remove the transaction from the pending lists and reset the account nonce
 	if pending := pool.pending[addr]; pending != nil {
 		if removed, invalids := pending.Remove(tx); removed {
@@ -1052,7 +1053,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Drop all transactions if they are less than the overflow
 			if size := uint64(list.Len()); size <= drop {
 				for _, tx := range list.Flatten() {
-					pool.removeTx(tx.Hash())
+					pool.removeTx(tx.Hash(), true)
 				}
 				drop -= size
 				queuedRateLimitCounter.Inc(int64(size))
@@ -1061,7 +1062,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Otherwise drop only last few transactions
 			txs := list.Flatten()
 			for i := len(txs) - 1; i >= 0 && drop > 0; i-- {
-				pool.removeTx(txs[i].Hash())
+				pool.removeTx(txs[i].Hash(), true)
 				drop--
 				queuedRateLimitCounter.Inc(1)
 			}

core/tx_pool_test.go

@@ -209,15 +209,10 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
 
 	pool.lockedReset(nil, nil)
 
-	pendingTx, err := pool.Pending()
+	_, err := pool.Pending()
 	if err != nil {
 		t.Fatalf("Could not fetch pending transactions: %v", err)
 	}
-
-	for addr, txs := range pendingTx {
-		t.Logf("%0x: %d\n", addr, len(txs))
-	}
-
 	nonce = pool.State().GetNonce(address)
 	if nonce != 2 {
 		t.Fatalf("Invalid nonce, want 2, got %d", nonce)
@@ -350,7 +345,7 @@ func TestTransactionChainFork(t *testing.T) {
 	if _, err := pool.add(tx, false); err != nil {
 		t.Error("didn't expect error", err)
 	}
-	pool.removeTx(tx.Hash())
+	pool.removeTx(tx.Hash(), true)
 
 	// reset the pool's internal state
 	resetState()
@@ -1388,13 +1383,13 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 		t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
 	// Ensure that adding high priced transactions drops cheap ones, but not own
-	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que -
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
 	pending, queued = pool.Stats()
@@ -1404,7 +1399,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 2 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
 	}
-	if err := validateEvents(events, 2); err != nil {
+	if err := validateEvents(events, 1); err != nil {
 		t.Fatalf("additional event firing failed: %v", err)
 	}
 	if err := validateTxPoolInternals(pool); err != nil {
@@ -1430,6 +1425,74 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	}
 }
 
+// Tests that more expensive transactions push out cheap ones from the pool, but
+// without producing instability by creating gaps that start jumping transactions
+// back and forth between queued/pending.
+func TestTransactionPoolStableUnderpricing(t *testing.T) {
+	t.Parallel()
+
+	// Create the pool to test the pricing enforcement with
+	db, _ := ethdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
+
+	config := testTxPoolConfig
+	config.GlobalSlots = 128
+	config.GlobalQueue = 0
+
+	pool := NewTxPool(config, params.TestChainConfig, blockchain)
+	defer pool.Stop()
+
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
+	// Create a number of test accounts and fund them
+	keys := make([]*ecdsa.PrivateKey, 2)
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
+	}
+	// Fill up the entire queue with the same transaction price points
+	txs := types.Transactions{}
+	for i := uint64(0); i < config.GlobalSlots; i++ {
+		txs = append(txs, pricedTransaction(i, 100000, big.NewInt(1), keys[0]))
+	}
+	pool.AddRemotes(txs)
+
+	pending, queued := pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, int(config.GlobalSlots)); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+	// Ensure that adding high priced transactions drops a cheap, but doesn't produce a gap
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+		t.Fatalf("failed to add well priced transaction: %v", err)
+	}
+	pending, queued = pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("additional event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+}
+
 // Tests that the pool rejects replacement transactions that don't meet the minimum
 // price bump required.
 func TestTransactionReplacement(t *testing.T) {

core/types/block.go

@@ -19,7 +19,6 @@ package types
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 	"math/big"
 	"sort"
@@ -389,40 +388,6 @@ func (b *Block) Hash() common.Hash {
 	return v
 }
 
-func (b *Block) String() string {
-	str := fmt.Sprintf(`Block(#%v): Size: %v {
-MinerHash: %x
-%v
-Transactions:
-%v
-Uncles:
-%v
-}
-`, b.Number(), b.Size(), b.header.HashNoNonce(), b.header, b.transactions, b.uncles)
-	return str
-}
-
-func (h *Header) String() string {
-	return fmt.Sprintf(`Header(%x):
-[
-	ParentHash:	    %x
-	UncleHash:	    %x
-	Coinbase:	    %x
-	Root:		    %x
-	TxSha		    %x
-	ReceiptSha:	    %x
-	Bloom:		    %x
-	Difficulty:	    %v
-	Number:		    %v
-	GasLimit:	    %v
-	GasUsed:	    %v
-	Time:		    %v
-	Extra:		    %s
-	MixDigest:      %x
-	Nonce:		    %x
-]`, h.Hash(), h.ParentHash, h.UncleHash, h.Coinbase, h.Root, h.TxHash, h.ReceiptHash, h.Bloom, h.Difficulty, h.Number, h.GasLimit, h.GasUsed, h.Time, h.Extra, h.MixDigest, h.Nonce)
-}
-
 type Blocks []*Block
 
 type BlockBy func(b1, b2 *Block) bool

core/types/log.go

@@ -17,7 +17,6 @@
 package types
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -95,10 +94,6 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
-func (l *Log) String() string {
-	return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
-}
-
 // LogForStorage is a wrapper around a Log that flattens and parses the entire content of
 // a log including non-consensus fields.
 type LogForStorage Log

core/types/receipt.go

@@ -149,14 +149,6 @@ func (r *Receipt) Size() common.StorageSize {
 	return size
 }
 
-// String implements the Stringer interface.
-func (r *Receipt) String() string {
-	if len(r.PostState) == 0 {
-		return fmt.Sprintf("receipt{status=%d cgas=%v bloom=%x logs=%v}", r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs)
-	}
-	return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
-}
-
 // ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
 // entire content of a receipt, as opposed to only the consensus fields originally.
 type ReceiptForStorage Receipt

core/types/transaction.go

@@ -19,7 +19,6 @@ package types
 import (
 	"container/heap"
 	"errors"
-	"fmt"
 	"io"
 	"math/big"
 	"sync/atomic"
@@ -34,7 +33,6 @@ import (
 
 var (
 	ErrInvalidSig = errors.New("invalid transaction v, r, s values")
-	errNoSigner   = errors.New("missing signing methods")
 )
 
 // deriveSigner makes a *best* guess about which signer to use.
@@ -262,58 +260,6 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
 	return tx.data.V, tx.data.R, tx.data.S
 }
 
-func (tx *Transaction) String() string {
-	var from, to string
-	if tx.data.V != nil {
-		// make a best guess about the signer and use that to derive
-		// the sender.
-		signer := deriveSigner(tx.data.V)
-		if f, err := Sender(signer, tx); err != nil { // derive but don't cache
-			from = "[invalid sender: invalid sig]"
-		} else {
-			from = fmt.Sprintf("%x", f[:])
-		}
-	} else {
-		from = "[invalid sender: nil V field]"
-	}
-
-	if tx.data.Recipient == nil {
-		to = "[contract creation]"
-	} else {
-		to = fmt.Sprintf("%x", tx.data.Recipient[:])
-	}
-	enc, _ := rlp.EncodeToBytes(&tx.data)
-	return fmt.Sprintf(`
-	TX(%x)
-	Contract: %v
-	From:     %s
-	To:       %s
-	Nonce:    %v
-	GasPrice: %#x
-	GasLimit  %#x
-	Value:    %#x
-	Data:     0x%x
-	V:        %#x
-	R:        %#x
-	S:        %#x
-	Hex:      %x
-`,
-		tx.Hash(),
-		tx.data.Recipient == nil,
-		from,
-		to,
-		tx.data.AccountNonce,
-		tx.data.Price,
-		tx.data.GasLimit,
-		tx.data.Amount,
-		tx.data.Payload,
-		tx.data.V,
-		tx.data.R,
-		tx.data.S,
-		enc,
-	)
-}
-
 // Transactions is a Transaction slice type for basic sorting.
 type Transactions []*Transaction
 

core/vm/instructions.go

@@ -31,7 +31,6 @@ import (
 var (
 	bigZero                  = new(big.Int)
 	tt255                    = math.BigPow(2, 255)
-	tt256                    = math.BigPow(2, 256)
 	errWriteProtection       = errors.New("evm: write protection")
 	errReturnDataOutOfBounds = errors.New("evm: return data out of bounds")
 	errExecutionReverted     = errors.New("evm: execution reverted")

core/vm/interpreter.go

@@ -42,7 +42,7 @@ type Config struct {
 }
 
 // Interpreter is used to run Ethereum based contracts and will utilise the
-// passed evmironment to query external sources for state information.
+// passed environment to query external sources for state information.
 // The Interpreter will run the byte code VM based on the passed
 // configuration.
 type Interpreter struct {
@@ -184,7 +184,7 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 			}
 		}
 		// consume the gas and return an error if not enough gas is available.
-		// cost is explicitly set so that the capture state defer method cas get the proper cost
+		// cost is explicitly set so that the capture state defer method can get the proper cost
 		cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize)
 		if err != nil || !contract.UseGas(cost) {
 			return nil, ErrOutOfGas

core/vm/logger.go

@@ -45,6 +45,7 @@ type LogConfig struct {
 	DisableMemory  bool // disable memory capture
 	DisableStack   bool // disable stack capture
 	DisableStorage bool // disable storage capture
+	Debug          bool // print output during capture end
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
@@ -184,6 +185,12 @@ func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost ui
 func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error {
 	l.output = output
 	l.err = err
+	if l.cfg.Debug {
+		fmt.Printf("0x%x\n", output)
+		if err != nil {
+			fmt.Printf(" error: %v\n", err)
+		}
+	}
 	return nil
 }
 

core/vm/runtime/runtime.go

@@ -103,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db))
 	}
 	var (
-		address = common.StringToAddress("contract")
+		address = common.BytesToAddress([]byte("contract"))
 		vmenv   = NewEnv(cfg)
 		sender  = vm.AccountRef(cfg.Origin)
 	)
@@ -113,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 	// Call the code with the given configuration.
 	ret, _, err := vmenv.Call(
 		sender,
-		common.StringToAddress("contract"),
+		common.BytesToAddress([]byte("contract")),
 		input,
 		cfg.GasLimit,
 		cfg.Value,

crypto/bn256/cloudflare/example_test.go

@@ -6,9 +6,12 @@ package bn256
 
 import (
 	"crypto/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
-func ExamplePair() {
+func TestExamplePair(t *testing.T) {
 	// This implements the tripartite Diffie-Hellman algorithm from "A One
 	// Round Protocol for Tripartite Diffie-Hellman", A. Joux.
 	// http://www.springerlink.com/content/cddc57yyva0hburb/fulltext.pdf
@@ -40,4 +43,9 @@ func ExamplePair() {
 	k3.ScalarMult(k3, c)
 
 	// k1, k2 and k3 will all be equal.
+
+	require.Equal(t, k1, k2)
+	require.Equal(t, k1, k3)
+
+	require.Equal(t, len(np), 4) //Avoid gometalinter varcheck err on np
 }

crypto/secp256k1/curve.go

@@ -290,11 +290,11 @@ func init() {
 	// See SEC 2 section 2.7.1
 	// curve parameters taken from:
 	// http://www.secg.org/collateral/sec2_final.pdf
-	theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
-	theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
-	theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
-	theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
-	theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
+	theCurve.P = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F")
+	theCurve.N = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141")
+	theCurve.B = math.MustParseBig256("0x0000000000000000000000000000000000000000000000000000000000000007")
+	theCurve.Gx = math.MustParseBig256("0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798")
+	theCurve.Gy = math.MustParseBig256("0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8")
 	theCurve.BitSize = 256
 }
 

eth/backend.go

@@ -63,7 +63,7 @@ type Ethereum struct {
 	chainConfig *params.ChainConfig
 
 	// Channel for shutting down the service
-	shutdownChan  chan bool    // Channel for shutting down the ethereum
+	shutdownChan  chan bool    // Channel for shutting down the Ethereum
 	stopDbUpgrade func() error // stop chain db sequential key upgrade
 
 	// Handlers
@@ -351,7 +351,7 @@ func (s *Ethereum) StartMining(local bool) error {
 	if local {
 		// If local (CPU) mining is started, we can disable the transaction rejection
 		// mechanism introduced to speed sync times. CPU mining on mainnet is ludicrous
-		// so noone will ever hit this path, whereas marking sync done on CPU mining
+		// so none will ever hit this path, whereas marking sync done on CPU mining
 		// will ensure that private networks work in single miner mode too.
 		atomic.StoreUint32(&s.protocolManager.acceptTxs, 1)
 	}

eth/db_upgrade.go

@@ -62,7 +62,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 			failed    error
 		)
 		for failed == nil && it.Next() {
-			// Skip any entries that don't look like old transaction meta entires (<hash>0x01)
+			// Skip any entries that don't look like old transaction meta entries (<hash>0x01)
 			key := it.Key()
 			if len(key) != common.HashLength+1 || key[common.HashLength] != 0x01 {
 				continue
@@ -86,7 +86,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 				}
 			}
 			// Convert the old metadata to a new lookup entry, delete duplicate data
-			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new looku entry
+			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new lookup entry
 				if failed = db.Delete(hash); failed == nil { // Delete the duplicate transaction data
 					if failed = db.Delete(append([]byte("receipts-"), hash...)); failed == nil { // Delete the duplicate receipt data
 						if failed = db.Delete(key); failed != nil { // Delete the old transaction metadata

eth/downloader/api.go

@@ -51,7 +51,7 @@ func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAP
 	return api
 }
 
-// eventLoop runs an loop until the event mux closes. It will install and uninstall new
+// eventLoop runs a loop until the event mux closes. It will install and uninstall new
 // sync subscriptions and broadcasts sync status updates to the installed sync subscriptions.
 func (api *PublicDownloaderAPI) eventLoop() {
 	var (

eth/downloader/downloader.go

@@ -47,7 +47,7 @@ var (
 
 	MaxForkAncestry  = 3 * params.EpochDuration // Maximum chain reorganisation
 	rttMinEstimate   = 2 * time.Second          // Minimum round-trip time to target for download requests
-	rttMaxEstimate   = 20 * time.Second         // Maximum rount-trip time to target for download requests
+	rttMaxEstimate   = 20 * time.Second         // Maximum round-trip time to target for download requests
 	rttMinConfidence = 0.1                      // Worse confidence factor in our estimated RTT value
 	ttlScaling       = 3                        // Constant scaling factor for RTT -> TTL conversion
 	ttlLimit         = time.Minute              // Maximum TTL allowance to prevent reaching crazy timeouts
@@ -135,9 +135,10 @@ type Downloader struct {
 	stateCh        chan dataPack // [eth/63] Channel receiving inbound node state data
 
 	// Cancellation and termination
-	cancelPeer string        // Identifier of the peer currently being used as the master (cancel on drop)
-	cancelCh   chan struct{} // Channel to cancel mid-flight syncs
-	cancelLock sync.RWMutex  // Lock to protect the cancel channel and peer in delivers
+	cancelPeer string         // Identifier of the peer currently being used as the master (cancel on drop)
+	cancelCh   chan struct{}  // Channel to cancel mid-flight syncs
+	cancelLock sync.RWMutex   // Lock to protect the cancel channel and peer in delivers
+	cancelWg   sync.WaitGroup // Make sure all fetcher goroutines have exited.
 
 	quitCh   chan struct{} // Quit channel to signal termination
 	quitLock sync.RWMutex  // Lock to prevent double closes
@@ -305,7 +306,7 @@ func (d *Downloader) UnregisterPeer(id string) error {
 	d.cancelLock.RUnlock()
 
 	if master {
-		d.Cancel()
+		d.cancel()
 	}
 	return nil
 }
@@ -476,12 +477,11 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 // spawnSync runs d.process and all given fetcher functions to completion in
 // separate goroutines, returning the first error that appears.
 func (d *Downloader) spawnSync(fetchers []func() error) error {
-	var wg sync.WaitGroup
 	errc := make(chan error, len(fetchers))
-	wg.Add(len(fetchers))
+	d.cancelWg.Add(len(fetchers))
 	for _, fn := range fetchers {
 		fn := fn
-		go func() { defer wg.Done(); errc <- fn() }()
+		go func() { defer d.cancelWg.Done(); errc <- fn() }()
 	}
 	// Wait for the first error, then terminate the others.
 	var err error
@@ -498,13 +498,13 @@ func (d *Downloader) spawnSync(fetchers []func() error) error {
 	}
 	d.queue.Close()
 	d.Cancel()
-	wg.Wait()
 	return err
 }
 
-// Cancel cancels all of the operations and resets the queue. It returns true
-// if the cancel operation was completed.
-func (d *Downloader) Cancel() {
+// cancel aborts all of the operations and resets the queue. However, cancel does
+// not wait for the running download goroutines to finish. This method should be
+// used when cancelling the downloads from inside the downloader.
+func (d *Downloader) cancel() {
 	// Close the current cancel channel
 	d.cancelLock.Lock()
 	if d.cancelCh != nil {
@@ -518,6 +518,13 @@ func (d *Downloader) Cancel() {
 	d.cancelLock.Unlock()
 }
 
+// Cancel aborts all of the operations and waits for all download goroutines to
+// finish before returning.
+func (d *Downloader) Cancel() {
+	d.cancel()
+	d.cancelWg.Wait()
+}
+
 // Terminate interrupts the downloader, canceling all pending operations.
 // The downloader cannot be reused after calling Terminate.
 func (d *Downloader) Terminate() {
@@ -884,7 +891,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64)
 // immediately to the header processor to keep the rest of the pipeline full even
 // in the case of header stalls.
 //
-// The method returs the entire filled skeleton and also the number of headers
+// The method returns the entire filled skeleton and also the number of headers
 // already forwarded for processing.
 func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ([]*types.Header, int, error) {
 	log.Debug("Filling up skeleton", "from", from)
@@ -1377,7 +1384,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 		pivot = height - uint64(fsMinFullBlocks)
 	}
 	// To cater for moving pivot points, track the pivot block and subsequently
-	// accumulated download results separatey.
+	// accumulated download results separately.
 	var (
 		oldPivot *fetchResult   // Locked in pivot block, might change eventually
 		oldTail  []*fetchResult // Downloaded content after the pivot
@@ -1615,7 +1622,7 @@ func (d *Downloader) qosReduceConfidence() {
 //
 // Note, the returned RTT is .9 of the actually estimated RTT. The reason is that
 // the downloader tries to adapt queries to the RTT, so multiple RTT values can
-// be adapted to, but smaller ones are preffered (stabler download stream).
+// be adapted to, but smaller ones are preferred (stabler download stream).
 func (d *Downloader) requestRTT() time.Duration {
 	return time.Duration(atomic.LoadUint64(&d.rttEstimate)) * 9 / 10
 }

eth/downloader/downloader_test.go

@@ -159,7 +159,7 @@ func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentRec
 	// Create the common suffix
 	hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false)
 
-	// Create the forks, making the second heavyer if non balanced forks were requested
+	// Create the forks, making the second heavier if non balanced forks were requested
 	hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false)
 	hashes1 = append(hashes1, hashes[1:]...)
 

eth/downloader/fakepeer.go

@@ -27,7 +27,7 @@ import (
 
 // FakePeer is a mock downloader peer that operates on a local database instance
 // instead of being an actual live node. It's useful for testing and to implement
-// sync commands from an xisting local database.
+// sync commands from an existing local database.
 type FakePeer struct {
 	id string
 	db ethdb.Database
@@ -48,7 +48,7 @@ func (p *FakePeer) Head() (common.Hash, *big.Int) {
 }
 
 // RequestHeadersByHash implements downloader.Peer, returning a batch of headers
-// defined by the origin hash and the associaed query parameters.
+// defined by the origin hash and the associated query parameters.
 func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header
@@ -92,7 +92,7 @@ func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int,
 }
 
 // RequestHeadersByNumber implements downloader.Peer, returning a batch of headers
-// defined by the origin number and the associaed query parameters.
+// defined by the origin number and the associated query parameters.
 func (p *FakePeer) RequestHeadersByNumber(number uint64, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header

eth/downloader/peer.go

@@ -551,7 +551,7 @@ func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peerC
 // medianRTT returns the median RTT of the peerset, considering only the tuning
 // peers if there are more peers available.
 func (ps *peerSet) medianRTT() time.Duration {
-	// Gather all the currnetly measured round trip times
+	// Gather all the currently measured round trip times
 	ps.lock.RLock()
 	defer ps.lock.RUnlock()
 

eth/downloader/queue.go

@@ -275,7 +275,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
 	if q.headerResults != nil {
 		panic("skeleton assembly already in progress")
 	}
-	// Shedule all the header retrieval tasks for the skeleton assembly
+	// Schedule all the header retrieval tasks for the skeleton assembly
 	q.headerTaskPool = make(map[uint64]*types.Header)
 	q.headerTaskQueue = prque.New()
 	q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains

eth/downloader/statesync.go

@@ -31,7 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-// stateReq represents a batch of state fetch requests groupped together into
+// stateReq represents a batch of state fetch requests grouped together into
 // a single data retrieval network packet.
 type stateReq struct {
 	items    []common.Hash              // Hashes of the state items to download
@@ -139,7 +139,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 
 		// Handle incoming state packs:
 		case pack := <-d.stateCh:
-			// Discard any data not requested (or previsouly timed out)
+			// Discard any data not requested (or previously timed out)
 			req := active[pack.PeerId()]
 			if req == nil {
 				log.Debug("Unrequested node data", "peer", pack.PeerId(), "len", pack.Items())
@@ -182,7 +182,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 		case req := <-d.trackStateReq:
 			// If an active request already exists for this peer, we have a problem. In
 			// theory the trie node schedule must never assign two requests to the same
-			// peer. In practive however, a peer might receive a request, disconnect and
+			// peer. In practice however, a peer might receive a request, disconnect and
 			// immediately reconnect before the previous times out. In this case the first
 			// request is never honored, alas we must not silently overwrite it, as that
 			// causes valid requests to go missing and sync to get stuck.
@@ -228,7 +228,7 @@ type stateSync struct {
 	err        error          // Any error hit during sync (set before completion)
 }
 
-// stateTask represents a single trie node download taks, containing a set of
+// stateTask represents a single trie node download task, containing a set of
 // peers already attempted retrieval from to detect stalled syncs and abort.
 type stateTask struct {
 	attempts map[string]struct{}
@@ -274,15 +274,21 @@ func (s *stateSync) Cancel() error {
 // receive data from peers, rather those are buffered up in the downloader and
 // pushed here async. The reason is to decouple processing from data receipt
 // and timeouts.
-func (s *stateSync) loop() error {
+func (s *stateSync) loop() (err error) {
 	// Listen for new peer events to assign tasks to them
 	newPeer := make(chan *peerConnection, 1024)
 	peerSub := s.d.peers.SubscribeNewPeers(newPeer)
 	defer peerSub.Unsubscribe()
+	defer func() {
+		cerr := s.commit(true)
+		if err == nil {
+			err = cerr
+		}
+	}()
 
 	// Keep assigning new tasks until the sync completes or aborts
 	for s.sched.Pending() > 0 {
-		if err := s.commit(false); err != nil {
+		if err = s.commit(false); err != nil {
 			return err
 		}
 		s.assignTasks()
@@ -307,14 +313,14 @@ func (s *stateSync) loop() error {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err := s.process(req); err != nil {
+			if err = s.process(req); err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
 			req.peer.SetNodeDataIdle(len(req.response))
 		}
 	}
-	return s.commit(true)
+	return nil
 }
 
 func (s *stateSync) commit(force bool) error {
@@ -323,7 +329,9 @@ func (s *stateSync) commit(force bool) error {
 	}
 	start := time.Now()
 	b := s.d.stateDB.NewBatch()
-	s.sched.Commit(b)
+	if written, err := s.sched.Commit(b); written == 0 || err != nil {
+		return err
+	}
 	if err := b.Write(); err != nil {
 		return fmt.Errorf("DB write error: %v", err)
 	}
@@ -333,7 +341,7 @@ func (s *stateSync) commit(force bool) error {
 	return nil
 }
 
-// assignTasks attempts to assing new tasks to all idle peers, either from the
+// assignTasks attempts to assign new tasks to all idle peers, either from the
 // batch currently being retried, or fetching new data from the trie sync itself.
 func (s *stateSync) assignTasks() {
 	// Iterate over all idle peers and try to assign them state fetches

eth/fetcher/fetcher.go

@@ -127,7 +127,7 @@ type Fetcher struct {
 	// Block cache
 	queue  *prque.Prque            // Queue containing the import operations (block number sorted)
 	queues map[string]int          // Per peer block counts to prevent memory exhaustion
-	queued map[common.Hash]*inject // Set of already queued blocks (to dedup imports)
+	queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports)
 
 	// Callbacks
 	getBlock       blockRetrievalFn   // Retrieves a block from the local chain

eth/fetcher/fetcher_test.go

@@ -198,7 +198,7 @@ func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*typ
 	}
 }
 
-// verifyFetchingEvent verifies that one single event arrive on an fetching channel.
+// verifyFetchingEvent verifies that one single event arrive on a fetching channel.
 func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) {
 	if arrive {
 		select {

eth/filters/api.go

@@ -98,7 +98,7 @@ func (api *PublicFilterAPI) timeoutLoop() {
 // NewPendingTransactionFilter creates a filter that fetches pending transaction hashes
 // as transactions enter the pending state.
 //
-// It is part of the filter package because this filter can be used throug the
+// It is part of the filter package because this filter can be used through the
 // `eth_getFilterChanges` polling method that is also used for log filters.
 //
 // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter

eth/filters/api_test.go

@@ -29,8 +29,8 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) {
 	var (
 		fromBlock rpc.BlockNumber = 0x123435
 		toBlock   rpc.BlockNumber = 0xabcdef
-		address0                  = common.StringToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
-		address1                  = common.StringToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
+		address0                  = common.HexToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
+		address1                  = common.HexToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
 		topic0                    = common.HexToHash("3ac225168df54212a25c1c01fd35bebfea408fdac2e31ddd6f80a4bbf9a5f1ca")
 		topic1                    = common.HexToHash("9084a792d2f8b16a62b882fd56f7860c07bf5fa91dd8a2ae7e809e5180fef0b3")
 		topic2                    = common.HexToHash("6ccae1c4af4152f460ff510e573399795dfab5dcf1fa60d1f33ac8fdc1e480ce")

eth/handler.go

@@ -96,8 +96,8 @@ type ProtocolManager struct {
 	wg sync.WaitGroup
 }
 
-// NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
-// with the ethereum network.
+// NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
+// with the Ethereum network.
 func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) {
 	// Create the protocol manager with the base fields
 	manager := &ProtocolManager{
@@ -498,20 +498,20 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			return errResp(ErrDecode, "msg %v: %v", msg, err)
 		}
 		// Deliver them all to the downloader for queuing
-		trasactions := make([][]*types.Transaction, len(request))
+		transactions := make([][]*types.Transaction, len(request))
 		uncles := make([][]*types.Header, len(request))
 
 		for i, body := range request {
-			trasactions[i] = body.Transactions
+			transactions[i] = body.Transactions
 			uncles[i] = body.Uncles
 		}
 		// Filter out any explicitly requested bodies, deliver the rest to the downloader
-		filter := len(trasactions) > 0 || len(uncles) > 0
+		filter := len(transactions) > 0 || len(uncles) > 0
 		if filter {
-			trasactions, uncles = pm.fetcher.FilterBodies(p.id, trasactions, uncles, time.Now())
+			transactions, uncles = pm.fetcher.FilterBodies(p.id, transactions, uncles, time.Now())
 		}
-		if len(trasactions) > 0 || len(uncles) > 0 || !filter {
-			err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
+		if len(transactions) > 0 || len(uncles) > 0 || !filter {
+			err := pm.downloader.DeliverBodies(p.id, transactions, uncles)
 			if err != nil {
 				log.Debug("Failed to deliver bodies", "err", err)
 			}

ethclient/ethclient.go

@@ -39,7 +39,11 @@ type Client struct {
 
 // Dial connects a client to the given URL.
 func Dial(rawurl string) (*Client, error) {
-	c, err := rpc.Dial(rawurl)
+	return DialContext(context.Background(), rawurl)
+}
+
+func DialContext(ctx context.Context, rawurl string) (*Client, error) {
+	c, err := rpc.DialContext(ctx, rawurl)
 	if err != nil {
 		return nil, err
 	}
@@ -51,6 +55,10 @@ func NewClient(c *rpc.Client) *Client {
 	return &Client{c}
 }
 
+func (ec *Client) Close() {
+	ec.c.Close()
+}
+
 // Blockchain Access
 
 // BlockByHash returns the given full block.
@@ -296,7 +304,7 @@ func (ec *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, err
 // SubscribeNewHead subscribes to notifications about the current blockchain head
 // on the given channel.
 func (ec *Client) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "newHeads", map[string]struct{}{})
+	return ec.c.EthSubscribe(ctx, ch, "newHeads")
 }
 
 // State Access

ethdb/database.go

@@ -17,6 +17,7 @@
 package ethdb
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,17 +33,25 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+const (
+	writeDelayNThreshold       = 200
+	writeDelayThreshold        = 350 * time.Millisecond
+	writeDelayWarningThrottler = 1 * time.Minute
+)
+
 var OpenFileLimit = 64
 
 type LDBDatabase struct {
 	fn string      // filename for reporting
 	db *leveldb.DB // LevelDB instance
 
-	compTimeMeter  metrics.Meter // Meter for measuring the total time spent in database compaction
-	compReadMeter  metrics.Meter // Meter for measuring the data read during compaction
-	compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
-	diskReadMeter  metrics.Meter // Meter for measuring the effective amount of data read
-	diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
+	compTimeMeter    metrics.Meter // Meter for measuring the total time spent in database compaction
+	compReadMeter    metrics.Meter // Meter for measuring the data read during compaction
+	compWriteMeter   metrics.Meter // Meter for measuring the data written during compaction
+	writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
+	writeDelayMeter  metrics.Meter // Meter for measuring the write delay duration due to database compaction
+	diskReadMeter    metrics.Meter // Meter for measuring the effective amount of data read
+	diskWriteMeter   metrics.Meter // Meter for measuring the effective amount of data written
 
 	quitLock sync.Mutex      // Mutex protecting the quit channel access
 	quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
@@ -91,9 +100,6 @@ func (db *LDBDatabase) Path() string {
 
 // Put puts the given key / value to the queue
 func (db *LDBDatabase) Put(key []byte, value []byte) error {
-	// Generate the data to write to disk, update the meter and write
-	//value = rle.Compress(value)
-
 	return db.db.Put(key, value, nil)
 }
 
@@ -103,18 +109,15 @@ func (db *LDBDatabase) Has(key []byte) (bool, error) {
 
 // Get returns the given key if it's present.
 func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
-	// Retrieve the key and increment the miss counter if not found
 	dat, err := db.db.Get(key, nil)
 	if err != nil {
 		return nil, err
 	}
 	return dat, nil
-	//return rle.Decompress(dat)
 }
 
 // Delete deletes the key from the queue and database
 func (db *LDBDatabase) Delete(key []byte) error {
-	// Execute the actual operation
 	return db.db.Delete(key, nil)
 }
 
@@ -153,16 +156,17 @@ func (db *LDBDatabase) LDB() *leveldb.DB {
 
 // Meter configures the database metrics collectors and
 func (db *LDBDatabase) Meter(prefix string) {
-	// Short circuit metering if the metrics system is disabled
-	if !metrics.Enabled {
-		return
+	if metrics.Enabled {
+		// Initialize all the metrics collector at the requested prefix
+		db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
+		db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
+		db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
+		db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil)
+		db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil)
 	}
-	// Initialize all the metrics collector at the requested prefix
-	db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
-	db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
-	db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
-	db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil)
-	db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil)
+	// Initialize write delay metrics no matter we are in metric mode or not.
+	db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil)
+	db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil)
 
 	// Create a quit channel for the periodic collector and run it
 	db.quitLock.Lock()
@@ -184,6 +188,9 @@ func (db *LDBDatabase) Meter(prefix string) {
 //      2   |        523 |    1000.37159 |       7.26059 |      66.86342 |      66.77884
 //      3   |        570 |    1113.18458 |       0.00000 |       0.00000 |       0.00000
 //
+// This is how the write delay look like (currently):
+// DelayN:5 Delay:406.604657ms
+//
 // This is how the iostats look like (currently):
 // Read(MB):3895.04860 Write(MB):3654.64712
 func (db *LDBDatabase) meter(refresh time.Duration) {
@@ -194,6 +201,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 	}
 	// Create storage for iostats.
 	var iostats [2]float64
+
+	// Create storage and warning log tracer for write delay.
+	var (
+		delaystats      [2]int64
+		lastWriteDelay  time.Time
+		lastWriteDelayN time.Time
+	)
+
 	// Iterate ad infinitum and collect the stats
 	for i := 1; ; i++ {
 		// Retrieve the database stats
@@ -242,6 +257,52 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 			db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024))
 		}
 
+		// Retrieve the write delay statistic
+		writedelay, err := db.db.GetProperty("leveldb.writedelay")
+		if err != nil {
+			db.log.Error("Failed to read database write delay statistic", "err", err)
+			return
+		}
+		var (
+			delayN        int64
+			delayDuration string
+			duration      time.Duration
+		)
+		if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s", &delayN, &delayDuration); n != 2 || err != nil {
+			db.log.Error("Write delay statistic not found")
+			return
+		}
+		duration, err = time.ParseDuration(delayDuration)
+		if err != nil {
+			db.log.Error("Failed to parse delay duration", "err", err)
+			return
+		}
+		if db.writeDelayNMeter != nil {
+			db.writeDelayNMeter.Mark(delayN - delaystats[0])
+			// If the write delay number been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			// If a warning that db performance is laggy has been displayed,
+			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
+			if int(db.writeDelayNMeter.Rate1()) > writeDelayNThreshold &&
+				time.Now().After(lastWriteDelayN.Add(writeDelayWarningThrottler)) {
+				db.log.Warn("Write delay number exceeds the threshold (200 per second) in the last minute")
+				lastWriteDelayN = time.Now()
+			}
+		}
+		if db.writeDelayMeter != nil {
+			db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1])
+			// If the write delay duration been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			// If a warning that db performance is laggy has been displayed,
+			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
+			if int64(db.writeDelayMeter.Rate1()) > writeDelayThreshold.Nanoseconds() &&
+				time.Now().After(lastWriteDelay.Add(writeDelayWarningThrottler)) {
+				db.log.Warn("Write delay duration exceeds the threshold (35% of the time) in the last minute")
+				lastWriteDelay = time.Now()
+			}
+		}
+		delaystats[0], delaystats[1] = delayN, duration.Nanoseconds()
+
 		// Retrieve the database iostats.
 		ioStats, err := db.db.GetProperty("leveldb.iostats")
 		if err != nil {

internal/ethapi/api.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -1388,7 +1389,7 @@ func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (strin
 	if block == nil {
 		return "", fmt.Errorf("block #%d not found", number)
 	}
-	return block.String(), nil
+	return spew.Sdump(block), nil
 }
 
 // SeedHash retrieves the seed hash of a block.

internal/jsre/deps/web3.js

@@ -2193,7 +2193,7 @@ var toWei = function(number, unit) {
 };
 
 /**
- * Takes an input and transforms it into an bignumber
+ * Takes an input and transforms it into a bignumber
  *
  * @method toBigNumber
  * @param {Number|String|BigNumber} a number, string, HEX string or BigNumber

les/bloombits.go

@@ -72,13 +72,3 @@ func (eth *LightEthereum) startBloomHandlers() {
 		}()
 	}
 }
-
-const (
-	// bloomConfirms is the number of confirmation blocks before a bloom section is
-	// considered probably final and its rotated bits are calculated.
-	bloomConfirms = 256
-
-	// bloomThrottling is the time to wait between processing two consecutive index
-	// sections. It's useful during chain upgrades to prevent disk overload.
-	bloomThrottling = 100 * time.Millisecond
-)

les/peer.go

@@ -545,9 +545,11 @@ func (ps *peerSet) notify(n peerSetNotify) {
 func (ps *peerSet) Register(p *peer) error {
 	ps.lock.Lock()
 	if ps.closed {
+		ps.lock.Unlock()
 		return errClosed
 	}
 	if _, ok := ps.peers[p.id]; ok {
+		ps.lock.Unlock()
 		return errAlreadyRegistered
 	}
 	ps.peers[p.id] = p

les/serverpool.go

@@ -73,7 +73,6 @@ const (
 	// and a short term value which is adjusted exponentially with a factor of
 	// pstatRecentAdjust with each dial/connection and also returned exponentially
 	// to the average with the time constant pstatReturnToMeanTC
-	pstatRecentAdjust   = 0.1
 	pstatReturnToMeanTC = time.Hour
 	// node address selection weight is dropped by a factor of exp(-addrFailDropLn) after
 	// each unsuccessful connection (restored after a successful one)
@@ -83,9 +82,6 @@ const (
 	responseScoreTC = time.Millisecond * 100
 	delayScoreTC    = time.Second * 5
 	timeoutPow      = 10
-	// peerSelectMinWeight is added to calculated weights at request peer selection
-	// to give poorly performing peers a little chance of coming back
-	peerSelectMinWeight = 0.005
 	// initStatsWeight is used to initialize previously unknown peers with good
 	// statistics to give a chance to prove themselves
 	initStatsWeight = 1

les/sync.go

@@ -25,11 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/light"
 )
 
-const (
-	//forceSyncCycle      = 10 * time.Second // Time interval to force syncs, even if few peers are available
-	minDesiredPeerCount = 5 // Amount of peers desired to start syncing
-)
-
 // syncer is responsible for periodically synchronising with the network, both
 // downloading hashes and blocks as well as handling the announcement handler.
 func (pm *ProtocolManager) syncer() {

light/postprocess.go

@@ -58,18 +58,18 @@ type trustedCheckpoint struct {
 var (
 	mainnetCheckpoint = trustedCheckpoint{
 		name:          "mainnet",
-		sectionIdx:    161,
-		sectionHead:   common.HexToHash("75b0c4baa7a62cece48abdcb03b6f31601961c9bece67dcd61df87aad4fc0d8d"),
-		chtRoot:       common.HexToHash("bbbfaa67b29716348997ec21a39c03b8d1fb973f6a43740b865595ba26ee812f"),
-		bloomTrieRoot: common.HexToHash("d6db6e6248354d7453391ce97830072a28ea4216be0bd95a5db9f53b1a64677b"),
+		sectionIdx:    165,
+		sectionHead:   common.HexToHash("21028acf9cd9ce80257221adc437c3c58ce046c4d43c21c3e9b1d1349059ec73"),
+		chtRoot:       common.HexToHash("26b2458cb7d0080d3a39311c914be92c368777a65ec074e1893b8bdc79e3910a"),
+		bloomTrieRoot: common.HexToHash("5d06908769179186165a72db7fc3473b25c28ed27efe78a392a9ff2c3fa67f84"),
 	}
 
 	ropstenCheckpoint = trustedCheckpoint{
 		name:          "ropsten",
-		sectionIdx:    83,
-		sectionHead:   common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"),
-		chtRoot:       common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"),
-		bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"),
+		sectionIdx:    92,
+		sectionHead:   common.HexToHash("21a158f9cc643da13a237cafceb37381072649f7278cf98c5820bfbced7cfcec"),
+		chtRoot:       common.HexToHash("1a8ddb8b086d7a33ca90eea90730225948fa504ae0283b15aff3c15c0e089bf9"),
+		bloomTrieRoot: common.HexToHash("fd192f92afbcdd0020c81ca0625116b5995509659653b10123bd986fe5129cc1"),
 	}
 )
 

miner/worker.go

@@ -117,6 +117,10 @@ type worker struct {
 	currentMu sync.Mutex
 	current   *Work
 
+	snapshotMu    sync.RWMutex
+	snapshotBlock *types.Block
+	snapshotState *state.StateDB
+
 	uncleMu        sync.Mutex
 	possibleUncles map[common.Hash]*types.Block
 
@@ -171,32 +175,28 @@ func (self *worker) setExtra(extra []byte) {
 }
 
 func (self *worker) pending() (*types.Block, *state.StateDB) {
+	if atomic.LoadInt32(&self.mining) == 0 {
+		// return a snapshot to avoid contention on currentMu mutex
+		self.snapshotMu.RLock()
+		defer self.snapshotMu.RUnlock()
+		return self.snapshotBlock, self.snapshotState.Copy()
+	}
+
 	self.currentMu.Lock()
 	defer self.currentMu.Unlock()
-
-	if atomic.LoadInt32(&self.mining) == 0 {
-		return types.NewBlock(
-			self.current.header,
-			self.current.txs,
-			nil,
-			self.current.receipts,
-		), self.current.state.Copy()
-	}
 	return self.current.Block, self.current.state.Copy()
 }
 
 func (self *worker) pendingBlock() *types.Block {
+	if atomic.LoadInt32(&self.mining) == 0 {
+		// return a snapshot to avoid contention on currentMu mutex
+		self.snapshotMu.RLock()
+		defer self.snapshotMu.RUnlock()
+		return self.snapshotBlock
+	}
+
 	self.currentMu.Lock()
 	defer self.currentMu.Unlock()
-
-	if atomic.LoadInt32(&self.mining) == 0 {
-		return types.NewBlock(
-			self.current.header,
-			self.current.txs,
-			nil,
-			self.current.receipts,
-		)
-	}
 	return self.current.Block
 }
 
@@ -268,6 +268,7 @@ func (self *worker) update() {
 				txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs)
 
 				self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase)
+				self.updateSnapshot()
 				self.currentMu.Unlock()
 			} else {
 				// If we're mining, but nothing is being processed, wake on new transactions
@@ -489,6 +490,7 @@ func (self *worker) commitNewWork() {
 		self.unconfirmed.Shift(work.Block.NumberU64() - 1)
 	}
 	self.push(work)
+	self.updateSnapshot()
 }
 
 func (self *worker) commitUncle(work *Work, uncle *types.Header) error {
@@ -506,6 +508,19 @@ func (self *worker) commitUncle(work *Work, uncle *types.Header) error {
 	return nil
 }
 
+func (self *worker) updateSnapshot() {
+	self.snapshotMu.Lock()
+	defer self.snapshotMu.Unlock()
+
+	self.snapshotBlock = types.NewBlock(
+		self.current.header,
+		self.current.txs,
+		nil,
+		self.current.receipts,
+	)
+	self.snapshotState = self.current.state.Copy()
+}
+
 func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) {
 	gp := new(core.GasPool).AddGas(env.header.GasLimit)
 

mobile/types.go

@@ -80,7 +80,7 @@ func (h *Header) EncodeRLP() ([]byte, error) {
 	return rlp.EncodeToBytes(h.header)
 }
 
-// NewHeaderFromJSON parses a header from an JSON data dump.
+// NewHeaderFromJSON parses a header from a JSON data dump.
 func NewHeaderFromJSON(data string) (*Header, error) {
 	h := &Header{
 		header: new(types.Header),
@@ -91,18 +91,12 @@ func NewHeaderFromJSON(data string) (*Header, error) {
 	return h, nil
 }
 
-// EncodeJSON encodes a header into an JSON data dump.
+// EncodeJSON encodes a header into a JSON data dump.
 func (h *Header) EncodeJSON() (string, error) {
 	data, err := json.Marshal(h.header)
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the header for debugging purposes.
-func (h *Header) String() string {
-	return h.header.String()
-}
-
 func (h *Header) GetParentHash() *Hash   { return &Hash{h.header.ParentHash} }
 func (h *Header) GetUncleHash() *Hash    { return &Hash{h.header.UncleHash} }
 func (h *Header) GetCoinbase() *Address  { return &Address{h.header.Coinbase} }
@@ -157,7 +151,7 @@ func (b *Block) EncodeRLP() ([]byte, error) {
 	return rlp.EncodeToBytes(b.block)
 }
 
-// NewBlockFromJSON parses a block from an JSON data dump.
+// NewBlockFromJSON parses a block from a JSON data dump.
 func NewBlockFromJSON(data string) (*Block, error) {
 	b := &Block{
 		block: new(types.Block),
@@ -168,18 +162,12 @@ func NewBlockFromJSON(data string) (*Block, error) {
 	return b, nil
 }
 
-// EncodeJSON encodes a block into an JSON data dump.
+// EncodeJSON encodes a block into a JSON data dump.
 func (b *Block) EncodeJSON() (string, error) {
 	data, err := json.Marshal(b.block)
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the block for debugging purposes.
-func (b *Block) String() string {
-	return b.block.String()
-}
-
 func (b *Block) GetParentHash() *Hash   { return &Hash{b.block.ParentHash()} }
 func (b *Block) GetUncleHash() *Hash    { return &Hash{b.block.UncleHash()} }
 func (b *Block) GetCoinbase() *Address  { return &Address{b.block.Coinbase()} }
@@ -232,7 +220,7 @@ func (tx *Transaction) EncodeRLP() ([]byte, error) {
 	return rlp.EncodeToBytes(tx.tx)
 }
 
-// NewTransactionFromJSON parses a transaction from an JSON data dump.
+// NewTransactionFromJSON parses a transaction from a JSON data dump.
 func NewTransactionFromJSON(data string) (*Transaction, error) {
 	tx := &Transaction{
 		tx: new(types.Transaction),
@@ -243,18 +231,12 @@ func NewTransactionFromJSON(data string) (*Transaction, error) {
 	return tx, nil
 }
 
-// EncodeJSON encodes a transaction into an JSON data dump.
+// EncodeJSON encodes a transaction into a JSON data dump.
 func (tx *Transaction) EncodeJSON() (string, error) {
 	data, err := json.Marshal(tx.tx)
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction for debugging purposes.
-func (tx *Transaction) String() string {
-	return tx.tx.String()
-}
-
 func (tx *Transaction) GetData() []byte      { return tx.tx.Data() }
 func (tx *Transaction) GetGas() int64        { return int64(tx.tx.Gas()) }
 func (tx *Transaction) GetGasPrice() *BigInt { return &BigInt{tx.tx.GasPrice()} }
@@ -330,7 +312,7 @@ func (r *Receipt) EncodeRLP() ([]byte, error) {
 	return rlp.EncodeToBytes(r.receipt)
 }
 
-// NewReceiptFromJSON parses a transaction receipt from an JSON data dump.
+// NewReceiptFromJSON parses a transaction receipt from a JSON data dump.
 func NewReceiptFromJSON(data string) (*Receipt, error) {
 	r := &Receipt{
 		receipt: new(types.Receipt),
@@ -341,18 +323,12 @@ func NewReceiptFromJSON(data string) (*Receipt, error) {
 	return r, nil
 }
 
-// EncodeJSON encodes a transaction receipt into an JSON data dump.
+// EncodeJSON encodes a transaction receipt into a JSON data dump.
 func (r *Receipt) EncodeJSON() (string, error) {
 	data, err := rlp.EncodeToBytes(r.receipt)
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction receipt for debugging purposes.
-func (r *Receipt) String() string {
-	return r.receipt.String()
-}
-
 func (r *Receipt) GetPostState() []byte         { return r.receipt.PostState }
 func (r *Receipt) GetCumulativeGasUsed() int64  { return int64(r.receipt.CumulativeGasUsed) }
 func (r *Receipt) GetBloom() *Bloom             { return &Bloom{r.receipt.Bloom} }

node/config.go

@@ -209,7 +209,7 @@ func DefaultHTTPEndpoint() string {
 	return config.HTTPEndpoint()
 }
 
-// WSEndpoint resolves an websocket endpoint based on the configured host interface
+// WSEndpoint resolves a websocket endpoint based on the configured host interface
 // and port parameters.
 func (c *Config) WSEndpoint() string {
 	if c.WSHost == "" {

node/node.go

@@ -303,50 +303,16 @@ func (n *Node) stopInProc() {
 
 // startIPC initializes and starts the IPC RPC endpoint.
 func (n *Node) startIPC(apis []rpc.API) error {
-	// Short circuit if the IPC endpoint isn't being exposed
 	if n.ipcEndpoint == "" {
-		return nil
+		return nil // IPC disabled.
 	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-			return err
-		}
-		n.log.Debug("IPC registered", "service", api.Service, "namespace", api.Namespace)
-	}
-	// All APIs registered, start the IPC listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = rpc.CreateIPCListener(n.ipcEndpoint); err != nil {
+	listener, handler, err := rpc.StartIPCEndpoint(n.ipcEndpoint, apis)
+	if err != nil {
 		return err
 	}
-	go func() {
-		n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint)
-
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				// Terminate if the listener was closed
-				n.lock.RLock()
-				closed := n.ipcListener == nil
-				n.lock.RUnlock()
-				if closed {
-					return
-				}
-				// Not closed, just some error; report and continue
-				n.log.Error("IPC accept failed", "err", err)
-				continue
-			}
-			go handler.ServeCodec(rpc.NewJSONCodec(conn), rpc.OptionMethodInvocation|rpc.OptionSubscriptions)
-		}
-	}()
-	// All listeners booted successfully
 	n.ipcListener = listener
 	n.ipcHandler = handler
-
+	n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint)
 	return nil
 }
 
@@ -370,30 +336,10 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 	if endpoint == "" {
 		return nil
 	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return err
-			}
-			n.log.Debug("HTTP registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
+	listener, handler, err := rpc.StartHTTPEndpoint(endpoint, apis, modules, cors, vhosts)
+	if err != nil {
 		return err
 	}
-	go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener)
 	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ","))
 	// All listeners booted successfully
 	n.httpEndpoint = endpoint
@@ -423,32 +369,11 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig
 	if endpoint == "" {
 		return nil
 	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return err
-			}
-			n.log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
+	listener, handler, err := rpc.StartWSEndpoint(endpoint, apis, modules, wsOrigins, exposeAll)
+	if err != nil {
 		return err
 	}
-	go rpc.NewWSServer(wsOrigins, handler).Serve(listener)
 	n.log.Info("WebSocket endpoint opened", "url", fmt.Sprintf("ws://%s", listener.Addr()))
-
 	// All listeners booted successfully
 	n.wsEndpoint = endpoint
 	n.wsListener = listener

p2p/discover/udp.go

@@ -49,7 +49,6 @@ var (
 // Timeouts
 const (
 	respTimeout = 500 * time.Millisecond
-	sendTimeout = 500 * time.Millisecond
 	expiration  = 20 * time.Second
 
 	ntpFailureThreshold = 32               // Continuous timeouts after which to check NTP

p2p/discv5/net.go

@@ -36,7 +36,6 @@ import (
 var (
 	errInvalidEvent = errors.New("invalid in current state")
 	errNoQuery      = errors.New("no pending query")
-	errWrongAddress = errors.New("unknown sender address")
 )
 
 const (
@@ -828,11 +827,10 @@ type nodeEvent uint
 //go:generate stringer -type=nodeEvent
 
 const (
-	invalidEvent nodeEvent = iota // zero is reserved
 
 	// Packet type events.
 	// These correspond to packet types in the UDP protocol.
-	pingPacket
+	pingPacket = iota + 1
 	pongPacket
 	findnodePacket
 	neighborsPacket

p2p/discv5/nodeevent_string.go

@@ -4,24 +4,14 @@ package discv5
 
 import "strconv"
 
-const (
-	_nodeEvent_name_0 = "invalidEventpingPacketpongPacketfindnodePacketneighborsPacketfindnodeHashPackettopicRegisterPackettopicQueryPackettopicNodesPacket"
-	_nodeEvent_name_1 = "pongTimeoutpingTimeoutneighboursTimeout"
-)
+const _nodeEvent_name = "pongTimeoutpingTimeoutneighboursTimeout"
 
-var (
-	_nodeEvent_index_0 = [...]uint8{0, 12, 22, 32, 46, 61, 79, 98, 114, 130}
-	_nodeEvent_index_1 = [...]uint8{0, 11, 22, 39}
-)
+var _nodeEvent_index = [...]uint8{0, 11, 22, 39}
 
 func (i nodeEvent) String() string {
-	switch {
-	case 0 <= i && i <= 8:
-		return _nodeEvent_name_0[_nodeEvent_index_0[i]:_nodeEvent_index_0[i+1]]
-	case 265 <= i && i <= 267:
-		i -= 265
-		return _nodeEvent_name_1[_nodeEvent_index_1[i]:_nodeEvent_index_1[i+1]]
-	default:
-		return "nodeEvent(" + strconv.FormatInt(int64(i), 10) + ")"
+	i -= 264
+	if i >= nodeEvent(len(_nodeEvent_index)-1) {
+		return "nodeEvent(" + strconv.FormatInt(int64(i+264), 10) + ")"
 	}
+	return _nodeEvent_name[_nodeEvent_index[i]:_nodeEvent_index[i+1]]
 }

p2p/discv5/table.go

@@ -38,7 +38,6 @@ const (
 	hashBits   = len(common.Hash{}) * 8
 	nBuckets   = hashBits + 1 // Number of buckets
 
-	maxBondingPingPongs = 16
 	maxFindnodeFailures = 5
 )