params: release Geth v1.8.4

* signer: introduce external signer command

* cmd/signer, rpc: Implement new signer. Add info about remote user to Context

* signer: refactored request/response, made use of urfave.cli

* cmd/signer: Use common flags

* cmd/signer: methods to validate calldata against abi

* cmd/signer: work on abi parser

* signer: add mutex around UI

* cmd/signer: add json 4byte directory, remove passwords from api

* cmd/signer: minor changes

* cmd/signer: Use ErrRequestDenied, enable lightkdf

* cmd/signer: implement tests

* cmd/signer: made possible for UI to modify tx parameters

* cmd/signer: refactors, removed channels in ui comms, added UI-api via stdin/out

* cmd/signer: Made lowercase json-definitions, added UI-signer test functionality

* cmd/signer: update documentation

* cmd/signer: fix bugs, improve abi detection, abi argument display

* cmd/signer: minor change in json format

* cmd/signer: rework json communication

* cmd/signer: implement mixcase addresses in API, fix json id bug

* cmd/signer: rename fromaccount, update pythonpoc with new json encoding format

* cmd/signer: make use of new abi interface

* signer: documentation

* signer/main: remove redundant  option

* signer: implement audit logging

* signer: create package 'signer', minor changes

* common: add 0x-prefix to mixcaseaddress in json marshalling + validation

* signer, rules, storage: implement rules + ephemeral storage for signer rules

* signer: implement OnApprovedTx, change signing response (API BREAKAGE)

* signer: refactoring + documentation

* signer/rules: implement dispatching to next handler

* signer: docs

* signer/rules: hide json-conversion from users, ensure context is cleaned

* signer: docs

* signer: implement validation rules, change signature of call_info

* signer: fix log flaw with string pointer

* signer: implement custom 4byte databsae that saves submitted signatures

* signer/storage: implement aes-gcm-backed credential storage

* accounts: implement json unmarshalling of url

* signer: fix listresponse, fix gas->uint64

* node: make http/ipc start methods public

* signer: add ipc capability+review concerns

* accounts: correct docstring

* signer: address review concerns

* rpc: go fmt -s

* signer: review concerns+ baptize Clef

* signer,node: move Start-functions to separate file

* signer: formatting

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sol linguist-language=Solidity

.travis.yml

@@ -12,34 +12,34 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: "1.10"
+      go: 1.10.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: "1.10"
+      go: 1.10.x
       script:
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - brew update
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: "1.10"
+      go: 1.10.x
       env:
         - lint
       git:
@@ -47,14 +47,12 @@ matrix:
       script:
         - go run build/ci.go lint
 
-    # This builder does the Ubuntu PPA and Linux Azure uploads
+    # This builder does the Ubuntu PPA upload
     - os: linux
       dist: trusty
-      sudo: required
-      go: "1.10"
+      go: 1.10.x
       env:
         - ubuntu-ppa
-        - azure-linux
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -63,11 +61,25 @@ matrix:
             - devscripts
             - debhelper
             - dput
-            - gcc-multilib
             - fakeroot
       script:
-        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+
+    # This builder does the Linux Azure uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.10.x
+      env:
+        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+      script:
+        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
         - go run build/ci.go install -arch 386
@@ -91,7 +103,7 @@ matrix:
       dist: trusty
       services:
         - docker
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-linux-mips
       git:
@@ -135,7 +147,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.10.1.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -152,7 +164,7 @@ matrix:
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-osx
         - azure-ios
@@ -181,8 +193,7 @@ matrix:
     # This builder does the Azure archive purges to avoid accumulating junk
     - os: linux
       dist: trusty
-      sudo: required
-      go: "1.10"
+      go: 1.10.x
       env:
         - azure-purge
       git:

Dockerfile

@@ -12,5 +12,11 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -12,4 +12,10 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp

README.md

@@ -142,7 +142,7 @@ Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containe
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
-network via your own programs and not manually through the console. To aid this, Geth has built in
+network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
 exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).

VERSION

@@ -1 +1 @@
-1.8.3
+1.8.4

accounts/abi/numbers.go

@@ -25,23 +25,23 @@ import (
 )
 
 var (
-	big_t      = reflect.TypeOf(&big.Int{})
-	derefbig_t = reflect.TypeOf(big.Int{})
-	uint8_t    = reflect.TypeOf(uint8(0))
-	uint16_t   = reflect.TypeOf(uint16(0))
-	uint32_t   = reflect.TypeOf(uint32(0))
-	uint64_t   = reflect.TypeOf(uint64(0))
-	int_t      = reflect.TypeOf(int(0))
-	int8_t     = reflect.TypeOf(int8(0))
-	int16_t    = reflect.TypeOf(int16(0))
-	int32_t    = reflect.TypeOf(int32(0))
-	int64_t    = reflect.TypeOf(int64(0))
-	address_t  = reflect.TypeOf(common.Address{})
-	int_ts     = reflect.TypeOf([]int(nil))
-	int8_ts    = reflect.TypeOf([]int8(nil))
-	int16_ts   = reflect.TypeOf([]int16(nil))
-	int32_ts   = reflect.TypeOf([]int32(nil))
-	int64_ts   = reflect.TypeOf([]int64(nil))
+	bigT      = reflect.TypeOf(&big.Int{})
+	derefbigT = reflect.TypeOf(big.Int{})
+	uint8T    = reflect.TypeOf(uint8(0))
+	uint16T   = reflect.TypeOf(uint16(0))
+	uint32T   = reflect.TypeOf(uint32(0))
+	uint64T   = reflect.TypeOf(uint64(0))
+	intT      = reflect.TypeOf(int(0))
+	int8T     = reflect.TypeOf(int8(0))
+	int16T    = reflect.TypeOf(int16(0))
+	int32T    = reflect.TypeOf(int32(0))
+	int64T    = reflect.TypeOf(int64(0))
+	addressT  = reflect.TypeOf(common.Address{})
+	intTS     = reflect.TypeOf([]int(nil))
+	int8TS    = reflect.TypeOf([]int8(nil))
+	int16TS   = reflect.TypeOf([]int16(nil))
+	int32TS   = reflect.TypeOf([]int32(nil))
+	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
@@ -52,7 +52,7 @@ func U256(n *big.Int) []byte {
 // checks whether the given reflect value is signed. This also works for slices with a number type
 func isSigned(v reflect.Value) bool {
 	switch v.Type() {
-	case int_ts, int8_ts, int16_ts, int32_ts, int64_ts, int_t, int8_t, int16_t, int32_t, int64_t:
+	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
 		return true
 	}
 	return false

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbigT {
 		return indirect(v.Elem())
 	}
 	return v
@@ -36,26 +36,26 @@ func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type)
 	switch size {
 	case 8:
 		if unsigned {
-			return reflect.Uint8, uint8_t
+			return reflect.Uint8, uint8T
 		}
-		return reflect.Int8, int8_t
+		return reflect.Int8, int8T
 	case 16:
 		if unsigned {
-			return reflect.Uint16, uint16_t
+			return reflect.Uint16, uint16T
 		}
-		return reflect.Int16, int16_t
+		return reflect.Int16, int16T
 	case 32:
 		if unsigned {
-			return reflect.Uint32, uint32_t
+			return reflect.Uint32, uint32T
 		}
-		return reflect.Int32, int32_t
+		return reflect.Int32, int32T
 	case 64:
 		if unsigned {
-			return reflect.Uint64, uint64_t
+			return reflect.Uint64, uint64T
 		}
-		return reflect.Int64, int64_t
+		return reflect.Int64, int64T
 	}
-	return reflect.Ptr, big_t
+	return reflect.Ptr, bigT
 }
 
 // mustArrayToBytesSlice creates a new byte slice with the exact same size as value

accounts/abi/type.go

@@ -135,7 +135,7 @@ func NewType(t string) (typ Type, err error) {
 		typ.Type = reflect.TypeOf(bool(false))
 	case "address":
 		typ.Kind = reflect.Array
-		typ.Type = address_t
+		typ.Type = addressT
 		typ.Size = 20
 		typ.T = AddressTy
 	case "string":

accounts/abi/type_test.go

@@ -46,36 +46,36 @@ func TestTypeRegexp(t *testing.T) {
 		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
 		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
 		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
 		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
 		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
 		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
@@ -84,9 +84,9 @@ func TestTypeRegexp(t *testing.T) {
 		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
 		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
 		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -252,6 +252,9 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes20", common.Address{}, ""},
 		{"address", [20]byte{}, ""},
 		{"address", common.Address{}, ""},
+		{"bytes32[]]", "", "invalid arg type in abi"},
+		{"invalidType", "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil && len(test.err) == 0 {

accounts/abi/unpack_test.go

@@ -56,6 +56,23 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: true,
 	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000000",
+		want: false,
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000001000000000001",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000003",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
 	{
 		def:  `[{"type": "uint32"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",

accounts/url.go

@@ -74,6 +74,22 @@ func (u URL) MarshalJSON() ([]byte, error) {
 	return json.Marshal(u.String())
 }
 
+// UnmarshalJSON parses url.
+func (u *URL) UnmarshalJSON(input []byte) error {
+	var textUrl string
+	err := json.Unmarshal(input, &textUrl)
+	if err != nil {
+		return err
+	}
+	url, err := parseURL(textUrl)
+	if err != nil {
+		return err
+	}
+	u.Scheme = url.Scheme
+	u.Path = url.Path
+	return nil
+}
+
 // Cmp compares x and y and returns:
 //
 //   -1 if x <  y

accounts/usbwallet/hub.go

@@ -127,7 +127,7 @@ func (hub *Hub) refreshWallets() {
 		// breaking the Ledger protocol if that is waiting for user confirmation. This
 		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
 		// need to prevent concurrent comms ourselves. The more elegant solution would
-		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// be to ditch enumeration in favor of hotplug events, but that don't work yet
 		// on Windows so if we need to hack it anyway, this is more elegant for now.
 		hub.commsLock.Lock()
 		if hub.commsPend > 0 { // A confirmation is pending, don't refresh

accounts/usbwallet/wallet.go

@@ -99,7 +99,7 @@ type wallet struct {
 	//
 	// As such, a hardware wallet needs two locks to function correctly. A state
 	// lock can be used to protect the wallet's software-side internal state, which
-	// must not be held exlusively during hardware communication. A communication
+	// must not be held exclusively during hardware communication. A communication
 	// lock can be used to achieve exclusive access to the device itself, this one
 	// however should allow "skipping" waiting for operations that might want to
 	// use the device, but can live without too (e.g. account self-derivation).

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.10.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

bmt/bmt.go

@@ -75,7 +75,7 @@ type Hasher struct {
 	blocksize   int         // segment size (size of hash) also for hash.Hash
 	count       int         // segment count
 	size        int         // for hash.Hash same as hashsize
-	cur         int         // cursor position for righmost currently open chunk
+	cur         int         // cursor position for rightmost currently open chunk
 	segment     []byte      // the rightmost open segment (not complete)
 	depth       int         // index of last level
 	result      chan []byte // result channel
@@ -149,7 +149,7 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
 	}
 }
 
-// Drain drains the pool uptil it has no more than n resources
+// Drain drains the pool until it has no more than n resources
 func (self *TreePool) Drain(n int) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
@@ -412,11 +412,10 @@ func (self *Hasher) Reset() {
 
 // ResetWithLength needs to be called before writing to the hasher
 // the argument is supposed to be the byte slice binary representation of
-// the legth of the data subsumed under the hash
+// the length of the data subsumed under the hash
 func (self *Hasher) ResetWithLength(l []byte) {
 	self.Reset()
 	self.blockLength = l
-
 }
 
 // Release gives back the Tree to the pool whereby it unlocks
@@ -531,7 +530,7 @@ func (self *Hasher) finalise(n *Node, i int) (d int) {
 	for {
 		// when the final segment's path is going via left segments
 		// the incoming data is pushed to the parent upon pulling the left
-		// we do not need toogle the state since this condition is
+		// we do not need toggle the state since this condition is
 		// detectable
 		n.unbalanced = isLeft
 		n.right = nil

build/ci.go

@@ -766,7 +766,7 @@ func doAndroidArchive(cmdline []string) {
 		if meta.Develop {
 			repo = *deploy + "/content/repositories/snapshots"
 		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
+		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
 			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
 			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
 	}

cmd/clef/README.md

@@ -0,0 +1,864 @@
+Clef
+----
+Clef can be used to sign transactions and data and is meant as a replacement for geth's account management.
+This allows DApps not to depend on geth's account management. When a DApp wants to sign data it can send the data to
+the signer, the signer will then provide the user with context and asks the user for permission to sign the data. If
+the users grants the signing request the signer will send the signature back to the DApp.
+  
+This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can
+help in situations when a DApp is connected to a remote node because a local Ethereum node is not available, not
+synchronised with the chain or a particular Ethereum node that has no built-in (or limited) account management.
+  
+Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
+or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
+
+
+## Command line flags
+Clef accepts the following command line options:
+```
+COMMANDS:
+   init    Initialize the signer, generate secret storage
+   attest  Attest that a js-file is to be used
+   addpw   Store a credential for a keystore file
+   help    Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --loglevel value        log level to emit to the screen (default: 4)
+   --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
+   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
+   --nousb                 Disables monitoring for and managing USB hardware wallets
+   --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcport value         HTTP-RPC server listening port (default: 8550)
+   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
+   --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
+   --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
+   --rules value           Enable rule-engine (default: "rules.json")
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --help, -h              show help
+   --version, -v           print the version
+
+```
+
+
+Example:
+```
+signer -keystore /my/keystore -chainid 4
+```
+
+Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works.
+
+## Security model
+
+The security model of the signer is as follows:
+
+* One critical component (the signer binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
+* The signer binary has a well-defined 'external' API.
+* The 'external' API is considered UNTRUSTED.
+* The signer binary also communicates with whatever process that invoked the binary, via stdin/stdout.
+  * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
+
+The general flow for signing a transaction using e.g. geth is as follows:
+![image](sign_flow.png)
+
+In this case, `geth` would be started with `--externalsigner=http://localhost:8550` and would relay requests to `eth.sendTransaction`.
+
+## TODOs
+
+Some snags and todos
+
+* [ ] The signer should take a startup param "--no-change", for UIs that do not contain the capability
+   to perform changes to things, only approve/deny. Such a UI should be able to start the signer in
+   a more secure mode by telling it that it only wants approve/deny capabilities.
+
+* [x] It would be nice if the signer could collect new 4byte-id:s/method selectors, and have a
+secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for
+inclusion upstream.
+
+* It should be possible to configure the signer to check if an account is indeed known to it, before
+passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate
+accounts if it immediately returned "unknown account".
+* [x] It should be possible to configure the signer to auto-allow listing (certain) accounts, instead of asking every time.
+* [x] Done Upon startup, the signer should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode),
+invoking methods with the following info:
+  * [x] Version info about the signer
+  * [x] Address of API (http/ipc)
+  * [ ] List of known accounts
+* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] `account_signRawTransaction`
+* [ ] `account_bulkSignTransactions([] transactions)` should
+   * only exist if enabled via config/flag
+   * only allow non-data-sending transactions
+   * all txs must use the same `from`-account
+   * let the user confirm, showing
+      * the total amount
+      * the number of unique recipients
+
+* Geth todos
+    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is
+put together is a bit of a hack into the http server. This could probably be greatly improved
+    - Relay: Geth should be started in `geth --external_signer localhost:8550`.
+    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This
+  type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which
+  retains the original input.
+    - The Geth api should switch to use the same type, and relay `to`-account verbatim to the external api.
+
+* [x] Storage
+    * [x] An encrypted key-value storage should be implemented
+    * See [rules.md](rules.md) for more info about this.
+
+* Another potential thing to introduce is pairing.
+  * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
+  * Thus geth/mist/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
+  * This feature would make the addition of rules less dangerous.
+
+* Wallets / accounts. Add API methods for wallets.
+
+## Communication
+
+### External API
+
+The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRPC standard as Geth. The messages are
+expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
+
+Some of these call can require user interaction. Clients must be aware that responses
+may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+
+The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
+that requests have any authority.
+
+### UI API
+
+The signer has one native console-based UI, for operation without any standalone tools.
+However, there is also an API to communicate with an external UI. To enable that UI,
+the signer needs to be executed with the `--stdio-ui` option, which allocates the
+`stdin`/`stdout` for the UI-api.
+
+An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
+
+The model is as follows:
+
+* The user starts the UI app (`pythonsigner.py`).
+* The UI app starts the `signer` with `--stdio-ui`, and listens to the
+process output for confirmation-requests.
+* The `signer` opens the external http api.
+* When the `signer` receives requests, it sends a `jsonrpc` request via `stdout`.
+* The UI app prompts the user accordingly, and responds to the `signer`
+* The `signer` signs (or not), and responds to the original request.
+
+## External API
+
+See the [external api changelog](extapi_changelog.md) for information about changes to this API.
+
+### Encoding
+- number: positive integers that are hex encoded
+- data: hex encoded data
+- string: ASCII string
+
+All hex encoded values must be prefixed with `0x`.
+
+## Methods
+
+### account_new
+
+#### Create new password protected account
+
+The signer will generate a new private key, encrypts it according to [web3 keystore spec](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) and stores it in the keystore directory.
+The client is responsible for creating a backup of the keystore. If the keystore is lost there is no method of retrieving lost accounts.
+
+#### Arguments
+
+None
+
+#### Result
+  - address [string]: account address that is derived from the generated key
+  - url [string]: location of the keyfile
+  
+#### Sample call
+```json
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "method": "account_new",
+  "params": []
+}
+
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+    "url": "keystore:///my/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+  }
+}
+```
+
+### account_list
+
+#### List available accounts
+   List all accounts that this signer currently manages
+
+#### Arguments
+
+None
+
+#### Result
+  - array with account records:
+     - account.address [string]: account address that is derived from the generated key
+     - account.type [string]: type of the 
+     - account.url [string]: location of the account
+  
+#### Sample call
+```json
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "method": "account_list"
+}
+
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "result": [
+    {
+      "address": "0xafb2f771f58513609765698f65d3f2f0224a956f",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T07-26-47.162109726Z--afb2f771f58513609765698f65d3f2f0224a956f"
+    },
+    {
+      "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+    }
+  ]
+}
+```
+
+### account_signTransaction
+
+#### Sign transactions
+   Signs a transactions and responds with the signed transaction in RLP encoded form.
+
+#### Arguments
+  2. transaction object:
+     - `from` [address]: account to send the transaction from
+     - `to` [address]: receiver account. If omitted or `0x`, will cause contract creation.
+     - `gas` [number]: maximum amount of gas to burn
+     - `gasPrice` [number]: gas price
+     - `value` [number:optional]: amount of Wei to send with the transaction
+     - `data` [data:optional]:  input data
+     - `nonce` [number]: account nonce
+  3. method signature [string:optional]
+     - The method signature, if present, is to aid decoding the calldata. Should consist of `methodname(paramtype,...)`, e.g. `transfer(uint256,address)`. The signer may use this data to parse the supplied calldata, and show the user. The data, however, is considered totally untrusted, and reliability is not expected.
+
+
+#### Result
+  - signed transaction in RLP encoded form [data]
+  
+#### Sample call
+```json
+{
+  "id": 2,
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+      "gas": "0x55555",
+      "gasPrice": "0x1234",
+      "input": "0xabcd",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x1234"
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "error": {
+    "code": -32000,
+    "message": "Request denied"
+  }
+}
+```
+#### Sample call with ABI-data
+
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+      "gas": "0x333",
+      "gasPrice": "0x1",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
+    },
+    "safeSend(address)"
+  ],
+  "id": 67
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "result": {
+    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+    "tx": {
+      "nonce": "0x0",
+      "gasPrice": "0x1",
+      "gas": "0x333",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+      "v": "0x26",
+      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
+      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
+    }
+  }
+}
+```
+
+Bash example:
+```bash
+#curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+
+{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
+```
+
+
+### account_sign
+
+#### Sign data
+   Signs a chunk of data and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [data]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "method": "account_sign",
+  "params": [
+    "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+    "0xaabbccdd"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "result": "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+}
+```
+
+### account_ecRecover
+
+#### Recover address
+   Derive the address from the account that was used to sign data from the data and signature.
+   
+#### Arguments
+  - data [data]: data that was signed
+  - signature [data]: the signature to verify
+
+#### Result
+  - derived account [address]
+  
+#### Sample call
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "method": "account_ecRecover",
+  "params": [
+    "0xaabbccdd",
+    "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
+}
+
+```
+
+### account_import
+
+#### Import account
+   Import a private key into the keystore. The imported key is expected to be encrypted according to the web3 keystore
+   format.
+   
+#### Arguments
+  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export) 
+
+#### Result
+  - imported key [object]:
+     - key.address [address]: address of the imported key
+     - key.type [string]: type of the account
+     - key.url [string]: key URL
+  
+#### Sample call
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "method": "account_import",
+  "params": [
+    {
+      "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+      "crypto": {
+        "cipher": "aes-128-ctr",
+        "cipherparams": {
+          "iv": "401c39a7c7af0388491c3d3ecb39f532"
+        },
+        "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+        "kdf": "scrypt",
+        "kdfparams": {
+          "dklen": 32,
+          "n": 262144,
+          "p": 1,
+          "r": 8,
+          "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+        },
+        "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+      },
+      "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+      "version": 3
+    },
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xc7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "type": "account",
+    "url": "keystore:///tmp/keystore/UTC--2017-08-24T11-00-42.032024108Z--c7412fc59930fd90099c917a50e5f11d0934b2f5"
+  }
+}
+```
+
+### account_export
+
+#### Export account from keystore
+   Export a private key from the keystore. The exported private key is encrypted with the original passphrase. When the
+   key is imported later this passphrase is required.
+   
+#### Arguments
+  - account [address]: export private key that is associated with this account
+
+#### Result
+  - exported key, see [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) for
+  more information
+  
+#### Sample call
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "method": "account_export",
+  "params": [
+    "0xc7412fc59930fd90099c917a50e5f11d0934b2f5"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "crypto": {
+      "cipher": "aes-128-ctr",
+      "cipherparams": {
+        "iv": "401c39a7c7af0388491c3d3ecb39f532"
+      },
+      "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+      "kdf": "scrypt",
+      "kdfparams": {
+        "dklen": 32,
+        "n": 262144,
+        "p": 1,
+        "r": 8,
+        "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+      },
+      "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+    },
+    "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+    "version": 3
+  }
+}
+```
+
+
+
+## UI API
+
+These methods needs to be implemented by a UI listener.
+
+By starting the signer with the switch `--stdio-ui-test`, the signer will invoke all known methods, and expect the UI to respond with
+denials. This can be used during development to ensure that the API is (at least somewhat) correctly implemented.
+See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to perform the 'denial-handshake-test'.
+
+All methods in this API uses object-based parameters, so that there can be no mixups of parameters: each piece of data is accessed by key.
+
+See the [ui api changelog](intapi_changelog.md) for information about changes to this API.
+
+OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
+Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
+things simpler for both parties.
+
+### ApproveTx
+
+Invoked when there's a transaction for approval.
+
+
+#### Sample call
+
+Here's a method invocation:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "Info",
+            "message": "safeSend(address: 0x0000000000000000000000000000000000000012)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48486",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+```
+
+The same method invocation, but with invalid data:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000002000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000002000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "WARNING",
+            "message": "Transaction data did not match ABI-interface: WARNING: Supplied data is stuffed with extra data. \nWant 0000000000000002000000000000000000000000000000000000000000000012\nHave 0000000000000000000000000000000000000000000000000000000000000012\nfor method safeSend(address)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48492",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+
+```
+
+One which has missing `to`, but with no `data`:
+
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 3,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "",
+        "to": null,
+        "gas": "0x0",
+        "gasPrice": "0x0",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": null,
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "CRITICAL",
+            "message": "Tx will create contract with empty code!"
+          }
+        ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+```
+
+### ApproveExport
+
+Invoked when a request to export an account has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 7,
+  "method": "ApproveExport",
+  "params": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ApproveListing
+
+Invoked when a request for account listing has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 5,
+  "method": "ApproveListing",
+  "params": [
+    {
+      "accounts": [
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-20T14-44-54.089682944Z--123409812340981234098123409812deadbeef42",
+          "address": "0x123409812340981234098123409812deadbeef42"
+        },
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-23T21-59-03.199240693Z--cafebabedeadbeef34098123409812deadbeef42",
+          "address": "0xcafebabedeadbeef34098123409812deadbeef42"
+        }
+      ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+
+### ApproveSignData
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 4,
+  "method": "ApproveSignData",
+  "params": [
+    {
+      "address": "0x123409812340981234098123409812deadbeef42",
+      "raw_data": "0x01020304",
+      "message": "\u0019Ethereum Signed Message:\n4\u0001\u0002\u0003\u0004",
+      "hash": "0x7e3a4e7a9d1744bc5c675c25e1234ca8ed9162bd17f78b9085e48047c15ac310",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ShowInfo
+
+The UI should show the info to the user. Does not expect response.
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 9,
+  "method": "ShowInfo",
+  "params": [
+    {
+      "text": "Tests completed"
+    }
+  ]
+}
+
+```
+
+### ShowError
+
+The UI should show the info to the user. Does not expect response.
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ShowError",
+  "params": [
+    {
+      "text": "Testing 'ShowError'"
+    }
+  ]
+}
+
+```
+
+### OnApproved
+
+`OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
+
+When implementing rate-limited rules, this callback should be used.
+
+TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
+
+### OnSignerStartup
+
+This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
+in k/v-form.
+
+Example call:
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+
+```
+
+
+### Rules for UI apis
+
+A UI should conform to the following rules.
+
+* A UI MUST NOT load any external resources that were not embedded/part of the UI package.
+  * For example, not load icons, stylesheets from the internet
+  * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
+* A Graphical UI MUST show the blocky-identicon for ethereum addresses.
+* A UI MUST warn display approproate warning if the destination-account is formatted with invalid checksum.
+* A UI MUST NOT open any ports or services
+  * The signer opens the public port
+* A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
+* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
+* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
+  * The signer provides accounts
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+along with the UI.
+
+

cmd/clef/extapi_changelog.md

@@ -0,0 +1,25 @@
+### Changelog for external API
+
+
+
+#### 2.0.0
+
+* Commit `73abaf04b1372fa4c43201fb1b8019fe6b0a6f8d`, move `from` into `transaction` object in `signTransaction`. This
+makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
+
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/intapi_changelog.md

@@ -0,0 +1,86 @@
+### Changelog for internal API (ui-api)
+
+### 2.0.0
+
+* Modify how `call_info` on a transaction is conveyed. New format:
+
+```
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x82A2A876D39022B3019932D30Cd9c97ad5616813",
+        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x123",
+        "value": "0x10",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+        {
+          "type": "WARNING",
+          "message": "Invalid checksum on to-address"
+        },
+        {
+          "type": "WARNING",
+          "message": "Tx contains data, but provided ABI signature could not be matched: Did not match: test (0 matches)"
+        }
+      ],
+      "meta": {
+        "remote": "127.0.0.1:54286",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+```
+
+#### 1.2.0
+
+* Add `OnStartup` method, to provide the UI with information about what API version
+the signer uses (both internal and external) aswell as build-info and external api.
+
+Example call:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+```
+
+#### 1.1.0
+
+* Add `OnApproved` method
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/main.go

@@ -0,0 +1,640 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// signer is a utility that can be used so sign transactions and
+// arbitrary data.
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"encoding/hex"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/rules"
+	"github.com/ethereum/go-ethereum/signer/storage"
+	"gopkg.in/urfave/cli.v1"
+	"os/signal"
+)
+
+// ExternalApiVersion -- see extapi_changelog.md
+const ExternalApiVersion = "2.0.0"
+
+// InternalApiVersion -- see intapi_changelog.md
+const InternalApiVersion = "2.0.0"
+
+const legalWarning = `
+WARNING! 
+
+Clef is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing. 
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE! 
+
+`
+
+var (
+	logLevelFlag = cli.IntFlag{
+		Name:  "loglevel",
+		Value: 4,
+		Usage: "log level to emit to the screen",
+	}
+	keystoreFlag = cli.StringFlag{
+		Name:  "keystore",
+		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
+		Usage: "Directory for the keystore",
+	}
+	configdirFlag = cli.StringFlag{
+		Name:  "configdir",
+		Value: DefaultConfigDir(),
+		Usage: "Directory for Clef configuration",
+	}
+	rpcPortFlag = cli.IntFlag{
+		Name:  "rpcport",
+		Usage: "HTTP-RPC server listening port",
+		Value: node.DefaultHTTPPort + 5,
+	}
+	signerSecretFlag = cli.StringFlag{
+		Name:  "signersecret",
+		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+	}
+	dBFlag = cli.StringFlag{
+		Name:  "4bytedb",
+		Usage: "File containing 4byte-identifiers",
+		Value: "./4byte.json",
+	}
+	customDBFlag = cli.StringFlag{
+		Name:  "4bytedb-custom",
+		Usage: "File used for writing new 4byte-identifiers submitted via API",
+		Value: "./4byte-custom.json",
+	}
+	auditLogFlag = cli.StringFlag{
+		Name:  "auditlog",
+		Usage: "File used to emit audit logs. Set to \"\" to disable",
+		Value: "audit.log",
+	}
+	ruleFlag = cli.StringFlag{
+		Name:  "rules",
+		Usage: "Enable rule-engine",
+		Value: "rules.json",
+	}
+	stdiouiFlag = cli.BoolFlag{
+		Name: "stdio-ui",
+		Usage: "Use STDIN/STDOUT as a channel for an external UI. " +
+			"This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " +
+			"interface, and can be used when Clef is started by an external process.",
+	}
+	testFlag = cli.BoolFlag{
+		Name:  "stdio-ui-test",
+		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
+	}
+	app         = cli.NewApp()
+	initCommand = cli.Command{
+		Action:    utils.MigrateFlags(initializeSecrets),
+		Name:      "init",
+		Usage:     "Initialize the signer, generate secret storage",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+		},
+		Description: `
+The init command generates a master seed which Clef can use to store credentials and data needed for 
+the rule-engine to work.`,
+	}
+	attestCommand = cli.Command{
+		Action:    utils.MigrateFlags(attestFile),
+		Name:      "attest",
+		Usage:     "Attest that a js-file is to be used",
+		ArgsUsage: "<sha256sum>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The attest command stores the sha256 of the rule.js-file that you want to use for automatic processing of 
+incoming requests. 
+
+Whenever you make an edit to the rule file, you need to use attestation to tell 
+Clef that the file is 'safe' to execute.`,
+	}
+
+	addCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(addCredential),
+		Name:      "addpw",
+		Usage:     "Store a credential for a keystore file",
+		ArgsUsage: "<address> <password>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+remove any stored credential for that address (keyfile)
+`,
+	}
+)
+
+func init() {
+	app.Name = "Clef"
+	app.Usage = "Manage Ethereum account operations"
+	app.Flags = []cli.Flag{
+		logLevelFlag,
+		keystoreFlag,
+		configdirFlag,
+		utils.NetworkIdFlag,
+		utils.LightKDFFlag,
+		utils.NoUSBFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCVirtualHostsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+		utils.RPCEnabledFlag,
+		rpcPortFlag,
+		signerSecretFlag,
+		dBFlag,
+		customDBFlag,
+		auditLogFlag,
+		ruleFlag,
+		stdiouiFlag,
+		testFlag,
+	}
+	app.Action = signer
+	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+
+}
+func main() {
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func initializeSecrets(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	configDir := c.String(configdirFlag.Name)
+
+	masterSeed := make([]byte, 256)
+	n, err := io.ReadFull(rand.Reader, masterSeed)
+	if err != nil {
+		return err
+	}
+	if n != len(masterSeed) {
+		return fmt.Errorf("failed to read enough random")
+	}
+	err = os.Mkdir(configDir, 0700)
+	if err != nil && !os.IsExist(err) {
+		return err
+	}
+	location := filepath.Join(configDir, "secrets.dat")
+	if _, err := os.Stat(location); err == nil {
+		return fmt.Errorf("file %v already exists, will not overwrite", location)
+	}
+	err = ioutil.WriteFile(location, masterSeed, 0700)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("A master seed has been generated into %s\n", location)
+	fmt.Printf(`
+This is required to be able to store credentials, such as : 
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it. 
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+
+`)
+	return nil
+}
+func attestFile(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+	// Initialize the encrypted storages
+	configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey)
+	val := ctx.Args().First()
+	configStorage.Put("ruleset_sha256", val)
+	log.Info("Ruleset attestation updated", "sha256", val)
+	return nil
+}
+
+func addCredential(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires at leaste one argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+
+	// Initialize the encrypted storages
+	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	key := ctx.Args().First()
+	value := ""
+	if len(ctx.Args()) > 1 {
+		value = ctx.Args().Get(1)
+	}
+	pwStorage.Put(key, value)
+	log.Info("Credential store updated", "key", key)
+	return nil
+}
+
+func initialize(c *cli.Context) error {
+	// Set up the logger to print everything
+	logOutput := os.Stdout
+	if c.Bool(stdiouiFlag.Name) {
+		logOutput = os.Stderr
+		// If using the stdioui, we can't do the 'confirm'-flow
+		fmt.Fprintf(logOutput, legalWarning)
+	} else {
+		if !confirm(legalWarning) {
+			return fmt.Errorf("aborted by user")
+		}
+	}
+
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(logOutput, log.TerminalFormat(true))))
+	return nil
+}
+
+func signer(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	var (
+		ui core.SignerUI
+	)
+	if c.Bool(stdiouiFlag.Name) {
+		log.Info("Using stdin/stdout as UI-channel")
+		ui = core.NewStdIOUI()
+	} else {
+		log.Info("Using CLI as UI-channel")
+		ui = core.NewCommandlineUI()
+	}
+	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+
+	var (
+		api core.ExternalAPI
+	)
+
+	configDir := c.String(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c); err != nil {
+		log.Info("No master seed provided, rules disabled")
+	} else {
+
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+
+		// Generate domain specific keys
+		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
+		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+		// Initialize the encrypted storages
+		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
+		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+
+		//Do we have a rule-file?
+		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		if err != nil {
+			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
+		} else {
+			hasher := sha256.New()
+			hasher.Write(ruleJS)
+			shasum := hasher.Sum(nil)
+			storedShasum := configStorage.Get("ruleset_sha256")
+			if storedShasum != hex.EncodeToString(shasum) {
+				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+			} else {
+				// Initialize rules
+				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
+				if err != nil {
+					utils.Fatalf(err.Error())
+				}
+				ruleEngine.Init(string(ruleJS))
+				ui = ruleEngine
+				log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
+			}
+		}
+	}
+
+	apiImpl := core.NewSignerAPI(
+		c.Int64(utils.NetworkIdFlag.Name),
+		c.String(keystoreFlag.Name),
+		c.Bool(utils.NoUSBFlag.Name),
+		ui, db,
+		c.Bool(utils.LightKDFFlag.Name))
+
+	api = apiImpl
+
+	// Audit logging
+	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+		api, err = core.NewAuditLogger(logfile, api)
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		log.Info("Audit logs configured", "file", logfile)
+	}
+	// register signer API with server
+	var (
+		extapiUrl = "n/a"
+		ipcApiUrl = "n/a"
+	)
+	rpcApi := []rpc.API{
+		{
+			Namespace: "account",
+			Public:    true,
+			Service:   api,
+			Version:   "1.0"},
+	}
+	if c.Bool(utils.RPCEnabledFlag.Name) {
+
+		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
+		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
+
+		// start http server
+		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts)
+		if err != nil {
+			utils.Fatalf("Could not start RPC api: %v", err)
+		}
+		extapiUrl = fmt.Sprintf("http://%s", httpEndpoint)
+		log.Info("HTTP endpoint opened", "url", extapiUrl)
+
+		defer func() {
+			listener.Close()
+			log.Info("HTTP endpoint closed", "url", httpEndpoint)
+		}()
+
+	}
+	if !c.Bool(utils.IPCDisabledFlag.Name) {
+		if c.IsSet(utils.IPCPathFlag.Name) {
+			ipcApiUrl = c.String(utils.IPCPathFlag.Name)
+		} else {
+			ipcApiUrl = filepath.Join(configDir, "clef.ipc")
+		}
+
+		listener, _, err := rpc.StartIPCEndpoint(func() bool { return true }, ipcApiUrl, rpcApi)
+		if err != nil {
+			utils.Fatalf("Could not start IPC api: %v", err)
+		}
+		log.Info("IPC endpoint opened", "url", ipcApiUrl)
+		defer func() {
+			listener.Close()
+			log.Info("IPC endpoint closed", "url", ipcApiUrl)
+		}()
+
+	}
+
+	if c.Bool(testFlag.Name) {
+		log.Info("Performing UI test")
+		go testExternalUI(apiImpl)
+	}
+	ui.OnSignerStartup(core.StartupInfo{
+		Info: map[string]interface{}{
+			"extapi_version": ExternalApiVersion,
+			"intapi_version": InternalApiVersion,
+			"extapi_http":    extapiUrl,
+			"extapi_ipc":     ipcApiUrl,
+		},
+	})
+
+	abortChan := make(chan os.Signal)
+	signal.Notify(abortChan, os.Interrupt)
+
+	sig := <-abortChan
+	log.Info("Exiting...", "signal", sig)
+
+	return nil
+}
+
+// splitAndTrim splits input separated by a comma
+// and trims excessive white space from the substrings.
+func splitAndTrim(input string) []string {
+	result := strings.Split(input, ",")
+	for i, r := range result {
+		result[i] = strings.TrimSpace(r)
+	}
+	return result
+}
+
+// DefaultConfigDir is the default config directory to use for the vaults and other
+// persistence requirements.
+func DefaultConfigDir() string {
+	// Try to place the data folder in the user's home dir
+	home := homeDir()
+	if home != "" {
+		if runtime.GOOS == "darwin" {
+			return filepath.Join(home, "Library", "Signer")
+		} else if runtime.GOOS == "windows" {
+			return filepath.Join(home, "AppData", "Roaming", "Signer")
+		} else {
+			return filepath.Join(home, ".clef")
+		}
+	}
+	// As we cannot guess a stable location, return empty and handle later
+	return ""
+}
+
+func homeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+	return ""
+}
+func readMasterKey(ctx *cli.Context) ([]byte, error) {
+	var (
+		file      string
+		configDir = ctx.String(configdirFlag.Name)
+	)
+	if ctx.IsSet(signerSecretFlag.Name) {
+		file = ctx.String(signerSecretFlag.Name)
+	} else {
+		file = filepath.Join(configDir, "secrets.dat")
+	}
+	if err := checkFile(file); err != nil {
+		return nil, err
+	}
+	masterKey, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	if len(masterKey) < 256 {
+		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	}
+	// Create vault location
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	err = os.Mkdir(vaultLocation, 0700)
+	if err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	//!TODO, use KDF to stretch the master key
+	//			stretched_key := stretch_key(master_key)
+
+	return masterKey, nil
+}
+
+// checkFile is a convenience function to check if a file
+// * exists
+// * is mode 0600
+func checkFile(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("failed stat on %s: %v", filename, err)
+	}
+	// Check the unix permission bits
+	if info.Mode().Perm()&077 != 0 {
+		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
+	}
+	return nil
+}
+
+// confirm displays a text and asks for user confirmation
+func confirm(text string) bool {
+	fmt.Printf(text)
+	fmt.Printf("\nEnter 'ok' to proceed:\n>")
+
+	text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+
+	if text := strings.TrimSpace(text); text == "ok" {
+		return true
+	}
+	return false
+}
+
+func testExternalUI(api *core.SignerAPI) {
+
+	ctx := context.WithValue(context.Background(), "remote", "clef binary")
+	ctx = context.WithValue(ctx, "scheme", "in-proc")
+	ctx = context.WithValue(ctx, "local", "main")
+
+	errs := make([]string, 0)
+
+	api.UI.ShowInfo("Testing 'ShowInfo'")
+	api.UI.ShowError("Testing 'ShowError'")
+
+	checkErr := func(method string, err error) {
+		if err != nil && err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+		}
+	}
+	var err error
+
+	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
+	checkErr("SignTransaction", err)
+	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+	checkErr("Sign", err)
+	_, err = api.List(ctx)
+	checkErr("List", err)
+	_, err = api.New(ctx)
+	checkErr("New", err)
+	_, err = api.Export(ctx, common.Address{})
+	checkErr("Export", err)
+	_, err = api.Import(ctx, json.RawMessage{})
+	checkErr("Import", err)
+
+	api.UI.ShowInfo("Tests completed")
+
+	if len(errs) > 0 {
+		log.Error("Got errors")
+		for _, e := range errs {
+			log.Error(e)
+		}
+	} else {
+		log.Info("No errors")
+	}
+
+}
+
+/**
+//Create Account
+
+curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_new","params":["test"],"id":67}' localhost:8550
+
+// List accounts
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_list","params":[""],"id":67}' http://localhost:8550/
+
+// Make Transaction
+// safeSend(0x12)
+// 4401a6e40000000000000000000000000000000000000000000000000000000000000012
+
+// supplied abi
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"test"],"id":67}' http://localhost:8550/
+
+// Not supplied
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"}],"id":67}' http://localhost:8550/
+
+// Sign data
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_sign","params":["0x694267f14675d7e1b9494fd8d72fefe1755710fa","bazonk gaz baz"],"id":67}' http://localhost:8550/
+
+
+**/

cmd/clef/pythonsigner.py

@@ -0,0 +1,179 @@
+import os,sys, subprocess
+from tinyrpc.transports import ServerTransport
+from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
+from tinyrpc.dispatch import public,RPCDispatcher
+from tinyrpc.server import RPCServer
+
+""" This is a POC example of how to write a custom UI for Clef. The UI starts the
+clef process with the '--stdio-ui' option, and communicates with clef using standard input / output.
+
+The standard input/output is a relatively secure way to communicate, as it does not require opening any ports
+or IPC files. Needless to say, it does not protect against memory inspection mechanisms where an attacker
+can access process memory."""
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib as urlparse
+
+class StdIOTransport(ServerTransport):
+    """ Uses std input/output for RPC """
+    def receive_message(self):
+        return None, urlparse.unquote(sys.stdin.readline())
+
+    def send_reply(self, context, reply):
+        print(reply)
+
+class PipeTransport(ServerTransport):
+    """ Uses std a pipe for RPC """
+
+    def __init__(self,input, output):
+        self.input = input
+        self.output = output
+
+    def receive_message(self):
+        data = self.input.readline()
+        print(">> {}".format( data))
+        return None, urlparse.unquote(data)
+
+    def send_reply(self, context, reply):
+        print("<< {}".format( reply))
+        self.output.write(reply)
+        self.output.write("\n")
+
+class StdIOHandler():
+
+    def __init__(self):
+        pass
+
+    @public
+    def ApproveTx(self,req):
+        """
+        Example request:
+        {
+            "jsonrpc": "2.0",
+            "method": "ApproveTx",
+            "params": [{
+                "transaction": {
+                    "to": "0xae967917c465db8578ca9024c205720b1a3651A9",
+                    "gas": "0x333",
+                    "gasPrice": "0x123",
+                    "value": "0x10",
+                    "data": "0xd7a5865800000000000000000000000000000000000000000000000000000000000000ff",
+                    "nonce": "0x0"
+                },
+                "from": "0xAe967917c465db8578ca9024c205720b1a3651A9",
+                "call_info": "Warning! Could not validate ABI-data against calldata\nSupplied ABI spec does not contain method signature in data: 0xd7a58658",
+                "meta": {
+                    "remote": "127.0.0.1:34572",
+                    "local": "localhost:8550",
+                    "scheme": "HTTP/1.1"
+                }
+            }],
+            "id": 1
+        }
+
+        :param transaction: transaction info
+        :param call_info: info abou the call, e.g. if ABI info could not be
+        :param meta: metadata about the request, e.g. where the call comes from
+        :return: 
+        """
+        transaction = req.get('transaction')
+        _from       = req.get('from')
+        call_info   = req.get('call_info')
+        meta        = req.get('meta')
+
+        return {
+            "approved" : False,
+            #"transaction" : transaction,
+  #          "from" : _from,
+#            "password" : None,
+        }
+
+    @public
+    def ApproveSignData(self, req):
+        """ Example request
+
+        """
+        return {"approved": False, "password" : None}
+
+    @public
+    def ApproveExport(self, req):
+        """ Example request
+
+        """
+        return {"approved" : False}
+
+    @public
+    def ApproveImport(self, req):
+        """ Example request
+
+        """
+        return { "approved" : False, "old_password": "", "new_password": ""}
+
+    @public
+    def ApproveListing(self, req):
+        """ Example request
+
+        """
+        return {'accounts': []}
+
+    @public
+    def ApproveNewAccount(self, req):
+        """
+        Example request
+
+        :return:
+        """
+        return {"approved": False,
+                #"password": ""
+                }
+
+    @public
+    def ShowError(self,message = {}):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowError'"},"id":1}
+
+        :param message: to show
+        :return: nothing
+        """
+        if 'text' in message.keys():
+            sys.stderr.write("Error: {}\n".format( message['text']))
+        return
+
+    @public
+    def ShowInfo(self,message = {}):
+        """
+        Example request
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowInfo'"},"id":0}
+
+        :param message: to display
+        :return:nothing
+        """
+
+        if 'text' in message.keys():
+            sys.stdout.write("Error: {}\n".format( message['text']))
+        return
+
+def main(args):
+
+    cmd = ["./clef", "--stdio-ui"]
+    if len(args) > 0 and args[0] == "test":
+        cmd.extend(["--stdio-ui-test"])
+    print("cmd: {}".format(" ".join(cmd)))
+    dispatcher = RPCDispatcher()
+    dispatcher.register_instance(StdIOHandler(), '')
+    # line buffered
+    p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    rpc_server = RPCServer(
+        PipeTransport(p.stdout, p.stdin),
+        JSONRPCProtocol(),
+        dispatcher
+    )
+    rpc_server.serve_forever()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

cmd/clef/rules.md

@@ -0,0 +1,236 @@
+# Rules
+
+The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
+
+It enables usecases like the following:
+
+* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
+* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`
+
+The two main features that are required for this to work well are;
+
+1. Rule Implementation: how to create, manage and interpret rules in a flexible but secure manner
+2. Credential managements and credentials; how to provide auto-unlock without exposing keys unnecessarily.
+
+The section below deals with both of them
+
+## Rule Implementation
+
+A ruleset file is implemented as a `js` file. Under the hood, the ruleset-engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
+defined in the UI protocol. Example:
+
+```javascript
+
+function asBig(str){
+    if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+    return new BigNumber(str)
+}
+
+// Approve transactions to a certain contract if value is below a certain limit
+function ApproveTx(req){
+
+    var limit = big.Newint("0xb1a2bc2ec50000")
+	var value = asBig(req.transaction.value);
+
+	if(req.transaction.to.toLowerCase()=="0xae967917c465db8578ca9024c205720b1a3651a9")
+	    && value.lt(limit) ){
+	    return "Approve"
+	 }
+    // If we return "Reject", it will be rejected.
+    // By not returning anything, it will be passed to the next UI, for manual processing
+}
+
+//Approve listings if request made from IPC
+function ApproveListing(req){
+    if (req.metadata.scheme == "ipc"){ return "Approve"}
+}
+
+```
+
+Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
+invokes the corresponding method. In doing so, there are three possible outcomes:
+
+1. JS returns "Approve"
+  * Auto-approve request
+2. JS returns "Reject"
+  * Auto-reject request
+3. Error occurs, or something else is returned
+  * Pass on to `next` ui: the regular UI channel.
+
+A more advanced example can be found below, "Example 1: ruleset for a rate-limited window", using `storage` to `Put` and `Get` `string`s by key.
+
+* At the time of writing, storage only exists as an ephemeral unencrypted implementation, to be used during testing.
+
+### Things to note
+
+The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
+
+* "use strict" will parse, but does nothing.
+* The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification.
+* Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported.
+
+Additionally, a few more have been added
+
+* The rule execution cannot load external javascript files.
+* The only preloaded libary is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
+* Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
+* Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
+* The JS engine has access to `storage` and `console`.
+
+#### Security considerations
+
+##### Security of ruleset
+
+Some security precautions can be made, such as:
+
+* Never load `ruleset.js` unless the file is `readonly` (`r-??-??-?`). If the user wishes to modify the ruleset, he must make it writeable and then set back to readonly.
+  * This is to prevent attacks where files are dropped on the users disk.
+* Since we're going to have to have some form of secure storage (not defined in this section), we could also store the `sha3` of the `ruleset.js` file in there.
+  * If the user wishes to modify the ruleset, he'd then have to perform e.g. `signer --attest /path/to/ruleset --credential <creds>`
+
+##### Security of implementation
+
+The drawbacks of this very flexible solution is that the `signer` needs to contain a javascript engine. This is pretty simple to implement, since it's already
+implemented for `geth`. There are no known security vulnerabilities in, nor have we had any security-problems with it so far.
+
+The javascript engine would be an added attack surface; but if the validation of `rulesets` is made good (with hash-based attestation), the actual javascript cannot be considered
+an attack surface -- if an attacker can control the ruleset, a much simpler attack would be to implement an "always-approve" rule instead of exploiting the js vm. The only benefit
+to be gained from attacking the actual `signer` process from the `js` side would be if it could somehow extract cryptographic keys from memory.
+
+##### Security in usability
+
+Javascript is flexible, but also easy to get wrong, especially when users assume that `js` can handle large integers natively. Typical errors
+include trying to multiply `gasCost` with `gas` without using `bigint`:s.
+
+It's unclear whether any other DSL could be more secure; since there's always the possibility of erroneously implementing a rule.
+
+
+## Credential management
+
+The ability to auto-approve transaction means that the signer needs to have necessary credentials to decrypt keyfiles. These passwords are hereafter called `ksp` (keystore pass).
+
+### Example implementation
+
+Upon startup of the signer, the signer is given a switch: `--seed <path/to/masterseed>`
+The `seed` contains a blob of bytes, which is the master seed for the `signer`.
+
+The `signer` uses the `seed` to:
+
+* Generate the `path` where the settings are stored.
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/vault.dat`
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/rules.js`
+* Generate the encryption password for `vault.dat`.
+
+The `vault.dat` would be an encrypted container storing the following information:
+
+* `ksp` entries
+* `sha256` hash of `rules.js`
+* Information about pair:ed callers (not yet specified)
+
+### Security considerations
+
+This would leave it up to the user to ensure that the `path/to/masterseed` is handled in a secure way. It's difficult to get around this, although one could
+imagine leveraging OS-level keychains where supported. The setup is however in general similar to how ssh-keys are  stored in `.ssh/`.
+
+
+# Implementation status
+
+This is now implemented (with ephemeral non-encrypted storage for now, so not yet enabled).
+
+## Example 1: ruleset for a rate-limited window
+
+
+```javascript
+
+	function big(str){
+		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str)
+	}
+
+	// Time window: 1 week
+	var window = 1000* 3600*24*7;
+
+	// Limit : 1 ether
+	var limit = new BigNumber("1e18");
+
+	function isLimitOk(transaction){
+		var value = big(transaction.value)
+		// Start of our window function
+		var windowstart = new Date().getTime() - window;
+
+		var txs = [];
+		var stored = storage.Get('txs');
+
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// First, remove all that have passed out of the time-window
+		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
+		console.log(txs, newtxs.length);
+
+		// Secondly, aggregate the current sum
+		sum = new BigNumber(0)
+
+		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+		console.log("ApproveTx > Sum so far", sum);
+		console.log("ApproveTx > Requested", value.toNumber());
+
+		// Would we exceed weekly limit ?
+		return sum.plus(value).lt(limit)
+
+	}
+	function ApproveTx(r){
+		if (isLimitOk(r.transaction)){
+			return "Approve"
+		}
+		return "Nope"
+	}
+
+	/**
+	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+ 	* 'response_str' contains the return value that will be sent to the external caller.
+	* The return value from this method is ignore - the reason for having this callback is to allow the
+	* ruleset to keep track of approved transactions.
+	*
+	* When implementing rate-limited rules, this callback should be used.
+	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+	* then accepts the transaction, this method will be called.
+	*
+	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+	*/
+ 	function OnApprovedTx(resp){
+		var value = big(resp.tx.value)
+		var txs = []
+		// Load stored transactions
+		var stored = storage.Get('txs');
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// Add this to the storage
+		txs.push({tstamp: new Date().getTime(), value: value});
+		storage.Put("txs", JSON.stringify(txs));
+	}
+
+```
+
+## Example 2: allow destination
+
+```javascript
+
+	function ApproveTx(r){
+		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
+		// Otherwise goes to manual processing
+	}
+
+```
+
+## Example 3: Allow listing
+
+```javascript
+
+    function ApproveListing(){
+        return "Approve"
+    }
+
+```

cmd/clef/tutorial.md

@@ -0,0 +1,198 @@
+## Initializing the signer
+
+First, initialize the master seed.
+
+```text
+#./signer init
+
+WARNING!
+
+The signer is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing.
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE!
+
+
+Enter 'ok' to proceed:
+>ok
+A master seed has been generated into /home/martin/.signer/secrets.dat
+
+This is required to be able to store credentials, such as :
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it.
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+```
+
+(for readability purposes, we'll remove the WARNING printout in the rest of this document)
+
+## Creating rules
+
+Now, you can create a rule-file.
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+```
+Get the `sha256` hash....
+```text
+#sha256sum rules.js
+6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
+```
+...And then `attest` the file:
+```text
+#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+
+INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+```
+At this point, we then start the signer with the rule-file:
+
+```text
+#./signer --rules rules.json
+
+INFO [02-21|12:15:18] Using CLI as UI-channel
+INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+DEBUG[02-21|12:15:18] Ledger support enabled
+DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+
+```
+
+Any list-requests will now be auto-approved by our rule-file.
+
+## Under the hood
+
+While doing the operations above, these files have been created:
+
+```text
+#ls -laR ~/.signer/
+/home/martin/.signer/:
+total 16
+drwx------  3 martin martin 4096 feb 21 12:14 .
+drwxr-xr-x 71 martin martin 4096 feb 21 12:12 ..
+drwx------  2 martin martin 4096 feb 21 12:14 43f73718397aa54d1b22
+-rwx------  1 martin martin  256 feb 21 12:12 secrets.dat
+
+/home/martin/.signer/43f73718397aa54d1b22:
+total 12
+drwx------ 2 martin martin 4096 feb 21 12:14 .
+drwx------ 3 martin martin 4096 feb 21 12:14 ..
+-rw------- 1 martin martin  159 feb 21 12:14 config.json
+
+#cat /home/martin/.signer/43f73718397aa54d1b22/config.json
+{"ruleset_sha256":{"iv":"6v4W4tfJxj3zZFbl","c":"6dt5RTDiTq93yh1qDEjpsat/tsKG7cb+vr3sza26IPL2fvsQ6ZoqFx++CPUa8yy6fD9Bbq41L01ehkKHTG3pOAeqTW6zc/+t0wv3AB6xPmU="}}
+
+```
+
+In `~/.signer`, the `secrets.dat` file was created, containing the `master_seed`.
+The `master_seed` was then used to derive a few other things:
+
+- `vault_location` : in this case `43f73718397aa54d1b22` .
+   - Thus, if you use a different `master_seed`, another `vault_location` will be used that does not conflict with each other.
+   - Example: `signer --signersecret /path/to/afile ...`
+- `config.json` which is the encrypted key/value storage for configuration data, containing the key `ruleset_sha256`.
+
+
+## Adding credentials
+
+In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+
+```text
+#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+
+INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
+```
+## More advanced rules
+
+Now let's update the rules to make use of credentials
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+function ApproveSignData(r){
+    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+    {
+        if(r.message.indexOf("bazonk") >= 0){
+            return "Approve"
+        }
+        return "Reject"
+    }
+    // Otherwise goes to manual processing
+}
+
+```
+In this example,
+* any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`,
+    * and auto-rejected if it does not.
+    * Any other signing-requests will be passed along for manual approve/reject.
+
+..attest the new file
+```text
+#sha256sum rules.js
+2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
+
+#./signer attest 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+
+INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+```
+
+And start the signer:
+
+```
+#./signer --rules rules.js
+
+INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[02-21|14:41:56] Trezor support enabled
+INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
+```
+And then test signing, once with `bazonk` and once without:
+
+```
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bazonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"result":"0x93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c"}
+
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"error":{"code":-32000,"message":"Request denied"}}
+
+```
+
+Meanwhile, in the signer output:
+```text
+INFO [02-21|14:42:41] Op approved
+INFO [02-21|14:42:56] Op rejected
+```
+
+The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
+
+```text
+#tail audit.log -n 4
+t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
+```

cmd/evm/runner.go

@@ -76,6 +76,7 @@ func runCmd(ctx *cli.Context) error {
 	logconfig := &vm.LogConfig{
 		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
 		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+		Debug:         ctx.GlobalBool(DebugFlag.Name),
 	}
 
 	var (
@@ -83,8 +84,8 @@ func runCmd(ctx *cli.Context) error {
 		debugLogger *vm.StructLogger
 		statedb     *state.StateDB
 		chainConfig *params.ChainConfig
-		sender      = common.StringToAddress("sender")
-		receiver    = common.StringToAddress("receiver")
+		sender      = common.BytesToAddress([]byte("sender"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -234,9 +235,7 @@ Gas used:           %d
 
 `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
 	}
-	if tracer != nil {
-		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime, err)
-	} else {
+	if tracer == nil {
 		fmt.Printf("0x%x\n", ret)
 		if err != nil {
 			fmt.Printf(" error: %v\n", err)

cmd/geth/bugcmd.go

@@ -49,15 +49,17 @@ func reportBug(ctx *cli.Context) error {
 	// execute template and write contents to buff
 	var buff bytes.Buffer
 
-	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "#### System information")
+	fmt.Fprintln(&buff)
 	fmt.Fprintln(&buff, "Version:", params.Version)
 	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
 	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
 	printOSDetails(&buff)
+	fmt.Fprintln(&buff, header)
 
 	// open a new GH issue
 	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
-		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+		fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String())
 	}
 	return nil
 }
@@ -97,13 +99,15 @@ func printCmdOut(w io.Writer, prefix, path string, args ...string) {
 	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
 }
 
-const header = `Please answer these questions before submitting your issue. Thanks!
+const header = `
+#### Expected behaviour
 
-#### What did you do?
- 
-#### What did you expect to see?
- 
-#### What did you see instead?
- 
-#### System details
+
+#### Actual behaviour
+
+
+#### Steps to reproduce the behaviour
+
+
+#### Backtrace
 `

cmd/geth/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -46,8 +45,6 @@ const (
 var (
 	// Git SHA1 commit hash of the release (set via linker flags)
 	gitCommit = ""
-	// Ethereum address of the Geth release oracle.
-	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
 	// flags that configure the node

cmd/puppeth/module_node.go

@@ -40,11 +40,11 @@ ADD genesis.json /genesis.json
 	ADD signer.pass /signer.pass
 {{end}}
 RUN \
-  echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
-	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+  echo 'geth --cache 512 init /genesis.json' > /root/geth.sh && \{{if .Unlock}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> /root/geth.sh && \{{end}}
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> /root/geth.sh
 
-ENTRYPOINT ["/bin/sh", "geth.sh"]
+ENTRYPOINT ["/bin/sh", "/root/geth.sh"]
 `
 
 // nodeComposefile is the docker-compose.yml file required to deploy and maintain

common/types.go

@@ -23,8 +23,10 @@ import (
 	"math/rand"
 	"reflect"
 
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"strings"
 )
 
 const (
@@ -45,9 +47,8 @@ func BytesToHash(b []byte) Hash {
 	h.SetBytes(b)
 	return h
 }
-func StringToHash(s string) Hash { return BytesToHash([]byte(s)) }
-func BigToHash(b *big.Int) Hash  { return BytesToHash(b.Bytes()) }
-func HexToHash(s string) Hash    { return BytesToHash(FromHex(s)) }
+func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) }
+func HexToHash(s string) Hash   { return BytesToHash(FromHex(s)) }
 
 // Get the string representation of the underlying hash
 func (h Hash) Str() string   { return string(h[:]) }
@@ -143,9 +144,8 @@ func BytesToAddress(b []byte) Address {
 	a.SetBytes(b)
 	return a
 }
-func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) }
-func BigToAddress(b *big.Int) Address  { return BytesToAddress(b.Bytes()) }
-func HexToAddress(s string) Address    { return BytesToAddress(FromHex(s)) }
+func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) }
+func HexToAddress(s string) Address   { return BytesToAddress(FromHex(s)) }
 
 // IsHexAddress verifies whether a string can represent a valid hex-encoded
 // Ethereum address or not.
@@ -240,3 +240,63 @@ func (a *UnprefixedAddress) UnmarshalText(input []byte) error {
 func (a UnprefixedAddress) MarshalText() ([]byte, error) {
 	return []byte(hex.EncodeToString(a[:])), nil
 }
+
+// MixedcaseAddress retains the original string, which may or may not be
+// correctly checksummed
+type MixedcaseAddress struct {
+	addr     Address
+	original string
+}
+
+// NewMixedcaseAddress constructor (mainly for testing)
+func NewMixedcaseAddress(addr Address) MixedcaseAddress {
+	return MixedcaseAddress{addr: addr, original: addr.Hex()}
+}
+
+// NewMixedcaseAddressFromString is mainly meant for unit-testing
+func NewMixedcaseAddressFromString(hexaddr string) (*MixedcaseAddress, error) {
+	if !IsHexAddress(hexaddr) {
+		return nil, fmt.Errorf("Invalid address")
+	}
+	a := FromHex(hexaddr)
+	return &MixedcaseAddress{addr: BytesToAddress(a), original: hexaddr}, nil
+}
+
+// UnmarshalJSON parses MixedcaseAddress
+func (ma *MixedcaseAddress) UnmarshalJSON(input []byte) error {
+	if err := hexutil.UnmarshalFixedJSON(addressT, input, ma.addr[:]); err != nil {
+		return err
+	}
+	return json.Unmarshal(input, &ma.original)
+}
+
+// MarshalJSON marshals the original value
+func (ma *MixedcaseAddress) MarshalJSON() ([]byte, error) {
+	if strings.HasPrefix(ma.original, "0x") || strings.HasPrefix(ma.original, "0X") {
+		return json.Marshal(fmt.Sprintf("0x%s", ma.original[2:]))
+	}
+	return json.Marshal(fmt.Sprintf("0x%s", ma.original))
+}
+
+// Address returns the address
+func (ma *MixedcaseAddress) Address() Address {
+	return ma.addr
+}
+
+// String implements fmt.Stringer
+func (ma *MixedcaseAddress) String() string {
+	if ma.ValidChecksum() {
+		return fmt.Sprintf("%s [chksum ok]", ma.original)
+	}
+	return fmt.Sprintf("%s [chksum INVALID]", ma.original)
+}
+
+// ValidChecksum returns true if the address has valid checksum
+func (ma *MixedcaseAddress) ValidChecksum() bool {
+	return ma.original == ma.addr.Hex()
+}
+
+// Original returns the mixed-case input string
+func (ma *MixedcaseAddress) Original() string {
+	return ma.original
+}

common/types_test.go

@@ -18,6 +18,7 @@ package common
 
 import (
 	"encoding/json"
+
 	"math/big"
 	"strings"
 	"testing"
@@ -149,3 +150,46 @@ func BenchmarkAddressHex(b *testing.B) {
 		testAddr.Hex()
 	}
 }
+
+func TestMixedcaseAccount_Address(t *testing.T) {
+
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md
+	// Note: 0X{checksum_addr} is not valid according to spec above
+
+	var res []struct {
+		A     MixedcaseAddress
+		Valid bool
+	}
+	if err := json.Unmarshal([]byte(`[
+		{"A" : "0xae967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0xAe967917c465db8578ca9024c205720b1a3651A9", "Valid": true},
+		{"A" : "0XAe967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0x1111111111111111111112222222222223333323", "Valid": true}
+		]`), &res); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, r := range res {
+		if got := r.A.ValidChecksum(); got != r.Valid {
+			t.Errorf("Expected checksum %v, got checksum %v, input %v", r.Valid, got, r.A.String())
+		}
+	}
+
+	//These should throw exceptions:
+	var r2 []MixedcaseAddress
+	for _, r := range []string{
+		`["0x11111111111111111111122222222222233333"]`,     // Too short
+		`["0x111111111111111111111222222222222333332"]`,    // Too short
+		`["0x11111111111111111111122222222222233333234"]`,  // Too long
+		`["0x111111111111111111111222222222222333332344"]`, // Too long
+		`["1111111111111111111112222222222223333323"]`,     // Missing 0x
+		`["x1111111111111111111112222222222223333323"]`,    // Missing 0
+		`["0xG111111111111111111112222222222223333323"]`,   //Non-hex
+	} {
+		if err := json.Unmarshal([]byte(r), &r2); err == nil {
+			t.Errorf("Expected failure, input %v", r)
+		}
+
+	}
+
+}

compression/rle/read_write.go

@@ -1,101 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package rle implements the run-length encoding used for Ethereum data.
-package rle
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-const (
-	token             byte = 0xfe
-	emptyShaToken          = 0xfd
-	emptyListShaToken      = 0xfe
-	tokenToken             = 0xff
-)
-
-var empty = crypto.Keccak256([]byte(""))
-var emptyList = crypto.Keccak256([]byte{0x80})
-
-func Decompress(dat []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-
-	for i := 0; i < len(dat); i++ {
-		if dat[i] == token {
-			if i+1 < len(dat) {
-				switch dat[i+1] {
-				case emptyShaToken:
-					buf.Write(empty)
-				case emptyListShaToken:
-					buf.Write(emptyList)
-				case tokenToken:
-					buf.WriteByte(token)
-				default:
-					buf.Write(make([]byte, int(dat[i+1]-2)))
-				}
-				i++
-			} else {
-				return nil, errors.New("error reading bytes. token encountered without proceeding bytes")
-			}
-		} else {
-			buf.WriteByte(dat[i])
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func compressChunk(dat []byte) (ret []byte, n int) {
-	switch {
-	case dat[0] == token:
-		return []byte{token, tokenToken}, 1
-	case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0:
-		j := 0
-		for j <= 254 && j < len(dat) {
-			if dat[j] != 0 {
-				break
-			}
-			j++
-		}
-		return []byte{token, byte(j + 2)}, j
-	case len(dat) >= 32:
-		if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) {
-			return []byte{token, emptyShaToken}, 32
-		} else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) {
-			return []byte{token, emptyListShaToken}, 32
-		}
-		fallthrough
-	default:
-		return dat[:1], 1
-	}
-}
-
-func Compress(dat []byte) []byte {
-	buf := new(bytes.Buffer)
-
-	i := 0
-	for i < len(dat) {
-		b, n := compressChunk(dat[i:])
-		buf.Write(b)
-		i += n
-	}
-
-	return buf.Bytes()
-}

compression/rle/read_write_test.go

@@ -1,50 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package rle
-
-import (
-	"testing"
-
-	checker "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { checker.TestingT(t) }
-
-type CompressionRleSuite struct{}
-
-var _ = checker.Suite(&CompressionRleSuite{})
-
-func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) {
-	exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70}
-	res, err := Decompress([]byte{token, 0xfd})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21}
-	res, err = Decompress([]byte{token, 0xfe})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	res, err = Decompress([]byte{token, 0xff})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, []byte{token})
-
-	res, err = Decompress([]byte{token, 12})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, make([]byte, 10))
-
-}

core/blockchain_test.go

@@ -1338,3 +1338,114 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		}
 	}
 }
+
+// Benchmarks large blocks with value transfers to non-existing accounts
+func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) {
+	var (
+		signer          = types.HomesteadSigner{}
+		testBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+		bankFunds       = big.NewInt(100000000000000000)
+		gspec           = Genesis{
+			Config: params.TestChainConfig,
+			Alloc: GenesisAlloc{
+				testBankAddress: {Balance: bankFunds},
+				common.HexToAddress("0xc0de"): {
+					Code:    []byte{0x60, 0x01, 0x50},
+					Balance: big.NewInt(0),
+				}, // push 1, pop
+			},
+			GasLimit: 100e6, // 100 M
+		}
+	)
+	// Generate the original common chain segment and the two competing forks
+	engine := ethash.NewFaker()
+	db, _ := ethdb.NewMemDatabase()
+	genesis := gspec.MustCommit(db)
+
+	blockGenerator := func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{1})
+		for txi := 0; txi < numTxs; txi++ {
+			uniq := uint64(i*numTxs + txi)
+			recipient := recipientFn(uniq)
+			//recipient := common.BigToAddress(big.NewInt(0).SetUint64(1337 + uniq))
+			tx, err := types.SignTx(types.NewTransaction(uniq, recipient, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testBankKey)
+			if err != nil {
+				b.Error(err)
+			}
+			block.AddTx(tx)
+		}
+	}
+
+	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, numBlocks, blockGenerator)
+	b.StopTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Import the shared chain and the original canonical one
+		diskdb, _ := ethdb.NewMemDatabase()
+		gspec.MustCommit(diskdb)
+
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+		if err != nil {
+			b.Fatalf("failed to create tester chain: %v", err)
+		}
+		b.StartTimer()
+		if _, err := chain.InsertChain(shared); err != nil {
+			b.Fatalf("failed to insert shared chain: %v", err)
+		}
+		b.StopTimer()
+		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
+			b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
+
+		}
+	}
+}
+func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}

core/database_util_test.go

@@ -317,7 +317,7 @@ func TestLookupStorage(t *testing.T) {
 			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
 				t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
 			}
-			if tx.String() != txn.String() {
+			if tx.Hash() != txn.Hash() {
 				t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
 			}
 		}

core/state/database.go

@@ -26,7 +26,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-// Trie cache generation limit after which to evic trie nodes from memory.
+// Trie cache generation limit after which to evict trie nodes from memory.
 var MaxTrieCacheGen = uint16(120)
 
 const (
@@ -151,9 +151,6 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 		return cached.(int), nil
 	}
 	code, err := db.ContractCode(addrHash, codeHash)
-	if err == nil {
-		db.codeSizeCache.Add(codeHash, len(code))
-	}
 	return len(code), err
 }
 

core/state/dump.go

@@ -53,7 +53,7 @@ func (self *StateDB) RawDump() Dump {
 			panic(err)
 		}
 
-		obj := newObject(nil, common.BytesToAddress(addr), data, nil)
+		obj := newObject(nil, common.BytesToAddress(addr), data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,

core/state/journal.go

@@ -22,11 +22,67 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// journalEntry is a modification entry in the state change journal that can be
+// reverted on demand.
 type journalEntry interface {
-	undo(*StateDB)
+	// revert undoes the changes introduced by this journal entry.
+	revert(*StateDB)
+
+	// dirtied returns the Ethereum address modified by this journal entry.
+	dirtied() *common.Address
 }
 
-type journal []journalEntry
+// journal contains the list of state modifications applied since the last state
+// commit. These are tracked to be able to be reverted in case of an execution
+// exception or revertal request.
+type journal struct {
+	entries []journalEntry         // Current changes tracked by the journal
+	dirties map[common.Address]int // Dirty accounts and the number of changes
+}
+
+// newJournal create a new initialized journal.
+func newJournal() *journal {
+	return &journal{
+		dirties: make(map[common.Address]int),
+	}
+}
+
+// append inserts a new modification entry to the end of the change journal.
+func (j *journal) append(entry journalEntry) {
+	j.entries = append(j.entries, entry)
+	if addr := entry.dirtied(); addr != nil {
+		j.dirties[*addr]++
+	}
+}
+
+// revert undoes a batch of journalled modifications along with any reverted
+// dirty handling too.
+func (j *journal) revert(statedb *StateDB, snapshot int) {
+	for i := len(j.entries) - 1; i >= snapshot; i-- {
+		// Undo the changes made by the operation
+		j.entries[i].revert(statedb)
+
+		// Drop any dirty tracking induced by the change
+		if addr := j.entries[i].dirtied(); addr != nil {
+			if j.dirties[*addr]--; j.dirties[*addr] == 0 {
+				delete(j.dirties, *addr)
+			}
+		}
+	}
+	j.entries = j.entries[:snapshot]
+}
+
+// dirty explicitly sets an address to dirty, even if the change entries would
+// otherwise suggest it as clean. This method is an ugly hack to handle the RIPEMD
+// precompile consensus exception.
+func (j *journal) dirty(addr common.Address) {
+	j.dirties[addr]++
+}
+
+// length returns the current number of entries in the journal.
+func (j *journal) length() int {
+	return len(j.entries)
+}
 
 type (
 	// Changes to the account trie.
@@ -77,16 +133,24 @@ type (
 	}
 )
 
-func (ch createObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) revert(s *StateDB) {
 	delete(s.stateObjects, *ch.account)
 	delete(s.stateObjectsDirty, *ch.account)
 }
 
-func (ch resetObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch resetObjectChange) revert(s *StateDB) {
 	s.setStateObject(ch.prev)
 }
 
-func (ch suicideChange) undo(s *StateDB) {
+func (ch resetObjectChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch suicideChange) revert(s *StateDB) {
 	obj := s.getStateObject(*ch.account)
 	if obj != nil {
 		obj.suicided = ch.prev
@@ -94,38 +158,60 @@ func (ch suicideChange) undo(s *StateDB) {
 	}
 }
 
-var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
-
-func (ch touchChange) undo(s *StateDB) {
-	if !ch.prev && *ch.account != ripemd {
-		s.getStateObject(*ch.account).touched = ch.prev
-		if !ch.prevDirty {
-			delete(s.stateObjectsDirty, *ch.account)
-		}
-	}
+func (ch suicideChange) dirtied() *common.Address {
+	return ch.account
 }
 
-func (ch balanceChange) undo(s *StateDB) {
+var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
+
+func (ch touchChange) revert(s *StateDB) {
+}
+
+func (ch touchChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch balanceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setBalance(ch.prev)
 }
 
-func (ch nonceChange) undo(s *StateDB) {
+func (ch balanceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch nonceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setNonce(ch.prev)
 }
 
-func (ch codeChange) undo(s *StateDB) {
+func (ch nonceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch codeChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
 }
 
-func (ch storageChange) undo(s *StateDB) {
+func (ch codeChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch storageChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setState(ch.key, ch.prevalue)
 }
 
-func (ch refundChange) undo(s *StateDB) {
+func (ch storageChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch refundChange) revert(s *StateDB) {
 	s.refund = ch.prev
 }
 
-func (ch addLogChange) undo(s *StateDB) {
+func (ch refundChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addLogChange) revert(s *StateDB) {
 	logs := s.logs[ch.txhash]
 	if len(logs) == 1 {
 		delete(s.logs, ch.txhash)
@@ -135,6 +221,14 @@ func (ch addLogChange) undo(s *StateDB) {
 	s.logSize--
 }
 
-func (ch addPreimageChange) undo(s *StateDB) {
+func (ch addLogChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addPreimageChange) revert(s *StateDB) {
 	delete(s.preimages, ch.hash)
 }
+
+func (ch addPreimageChange) dirtied() *common.Address {
+	return nil
+}

core/state/state_object.go

@@ -85,9 +85,7 @@ type stateObject struct {
 	// during the "update" phase of the state transition.
 	dirtyCode bool // true if the code was updated
 	suicided  bool
-	touched   bool
 	deleted   bool
-	onDirty   func(addr common.Address) // Callback method to mark a state object newly dirty
 }
 
 // empty returns whether the account is considered empty.
@@ -105,7 +103,7 @@ type Account struct {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, data Account, onDirty func(addr common.Address)) *stateObject {
+func newObject(db *StateDB, address common.Address, data Account) *stateObject {
 	if data.Balance == nil {
 		data.Balance = new(big.Int)
 	}
@@ -119,7 +117,6 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a
 		data:          data,
 		cachedStorage: make(Storage),
 		dirtyStorage:  make(Storage),
-		onDirty:       onDirty,
 	}
 }
 
@@ -137,23 +134,17 @@ func (self *stateObject) setError(err error) {
 
 func (self *stateObject) markSuicided() {
 	self.suicided = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (c *stateObject) touch() {
-	c.db.journal = append(c.db.journal, touchChange{
-		account:   &c.address,
-		prev:      c.touched,
-		prevDirty: c.onDirty == nil,
+	c.db.journal.append(touchChange{
+		account: &c.address,
 	})
-	if c.onDirty != nil {
-		c.onDirty(c.Address())
-		c.onDirty = nil
+	if c.address == ripemd {
+		// Explicitly put it in the dirty-cache, which is otherwise generated from
+		// flattened journals.
+		c.db.journal.dirty(c.address)
 	}
-	c.touched = true
 }
 
 func (c *stateObject) getTrie(db Database) Trie {
@@ -195,7 +186,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash {
 
 // SetState updates a value in account storage.
 func (self *stateObject) SetState(db Database, key, value common.Hash) {
-	self.db.journal = append(self.db.journal, storageChange{
+	self.db.journal.append(storageChange{
 		account:  &self.address,
 		key:      key,
 		prevalue: self.GetState(db, key),
@@ -207,10 +198,6 @@ func (self *stateObject) setState(key, value common.Hash) {
 	self.cachedStorage[key] = value
 	self.dirtyStorage[key] = value
 
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // updateTrie writes cached storage modifications into the object's storage trie.
@@ -274,7 +261,7 @@ func (c *stateObject) SubBalance(amount *big.Int) {
 }
 
 func (self *stateObject) SetBalance(amount *big.Int) {
-	self.db.journal = append(self.db.journal, balanceChange{
+	self.db.journal.append(balanceChange{
 		account: &self.address,
 		prev:    new(big.Int).Set(self.data.Balance),
 	})
@@ -283,17 +270,13 @@ func (self *stateObject) SetBalance(amount *big.Int) {
 
 func (self *stateObject) setBalance(amount *big.Int) {
 	self.data.Balance = amount
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // Return the gas back to the origin. Used by the Virtual machine or Closures
 func (c *stateObject) ReturnGas(gas *big.Int) {}
 
-func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject {
-	stateObject := newObject(db, self.address, self.data, onDirty)
+func (self *stateObject) deepCopy(db *StateDB) *stateObject {
+	stateObject := newObject(db, self.address, self.data)
 	if self.trie != nil {
 		stateObject.trie = db.db.CopyTrie(self.trie)
 	}
@@ -333,7 +316,7 @@ func (self *stateObject) Code(db Database) []byte {
 
 func (self *stateObject) SetCode(codeHash common.Hash, code []byte) {
 	prevcode := self.Code(self.db.db)
-	self.db.journal = append(self.db.journal, codeChange{
+	self.db.journal.append(codeChange{
 		account:  &self.address,
 		prevhash: self.CodeHash(),
 		prevcode: prevcode,
@@ -345,14 +328,10 @@ func (self *stateObject) setCode(codeHash common.Hash, code []byte) {
 	self.code = code
 	self.data.CodeHash = codeHash[:]
 	self.dirtyCode = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) SetNonce(nonce uint64) {
-	self.db.journal = append(self.db.journal, nonceChange{
+	self.db.journal.append(nonceChange{
 		account: &self.address,
 		prev:    self.data.Nonce,
 	})
@@ -361,10 +340,6 @@ func (self *stateObject) SetNonce(nonce uint64) {
 
 func (self *stateObject) setNonce(nonce uint64) {
 	self.data.Nonce = nonce
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) CodeHash() []byte {

core/state/statedb.go

@@ -76,7 +76,7 @@ type StateDB struct {
 
 	// Journal of state modifications. This is the backbone of
 	// Snapshot and RevertToSnapshot.
-	journal        journal
+	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
 
@@ -96,6 +96,7 @@ func New(root common.Hash, db Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		logs:              make(map[common.Hash][]*types.Log),
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}, nil
 }
 
@@ -131,7 +132,7 @@ func (self *StateDB) Reset(root common.Hash) error {
 }
 
 func (self *StateDB) AddLog(log *types.Log) {
-	self.journal = append(self.journal, addLogChange{txhash: self.thash})
+	self.journal.append(addLogChange{txhash: self.thash})
 
 	log.TxHash = self.thash
 	log.BlockHash = self.bhash
@@ -156,7 +157,7 @@ func (self *StateDB) Logs() []*types.Log {
 // AddPreimage records a SHA3 preimage seen by the VM.
 func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) {
 	if _, ok := self.preimages[hash]; !ok {
-		self.journal = append(self.journal, addPreimageChange{hash: hash})
+		self.journal.append(addPreimageChange{hash: hash})
 		pi := make([]byte, len(preimage))
 		copy(pi, preimage)
 		self.preimages[hash] = pi
@@ -169,7 +170,7 @@ func (self *StateDB) Preimages() map[common.Hash][]byte {
 }
 
 func (self *StateDB) AddRefund(gas uint64) {
-	self.journal = append(self.journal, refundChange{prev: self.refund})
+	self.journal.append(refundChange{prev: self.refund})
 	self.refund += gas
 }
 
@@ -235,10 +236,10 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash {
 	return common.BytesToHash(stateObject.CodeHash())
 }
 
-func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) GetState(addr common.Address, bhash common.Hash) common.Hash {
+	stateObject := self.getStateObject(addr)
 	if stateObject != nil {
-		return stateObject.GetState(self.db, b)
+		return stateObject.GetState(self.db, bhash)
 	}
 	return common.Hash{}
 }
@@ -250,12 +251,12 @@ func (self *StateDB) Database() Database {
 
 // StorageTrie returns the storage trie of an account.
 // The return value is a copy and is nil for non-existent accounts.
-func (self *StateDB) StorageTrie(a common.Address) Trie {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) StorageTrie(addr common.Address) Trie {
+	stateObject := self.getStateObject(addr)
 	if stateObject == nil {
 		return nil
 	}
-	cpy := stateObject.deepCopy(self, nil)
+	cpy := stateObject.deepCopy(self)
 	return cpy.updateTrie(self.db)
 }
 
@@ -271,7 +272,7 @@ func (self *StateDB) HasSuicided(addr common.Address) bool {
  * SETTERS
  */
 
-// AddBalance adds amount to the account associated with addr
+// AddBalance adds amount to the account associated with addr.
 func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -279,7 +280,7 @@ func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	}
 }
 
-// SubBalance subtracts amount from the account associated with addr
+// SubBalance subtracts amount from the account associated with addr.
 func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -308,7 +309,7 @@ func (self *StateDB) SetCode(addr common.Address, code []byte) {
 	}
 }
 
-func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
+func (self *StateDB) SetState(addr common.Address, key, value common.Hash) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
 		stateObject.SetState(self.db, key, value)
@@ -325,7 +326,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 	if stateObject == nil {
 		return false
 	}
-	self.journal = append(self.journal, suicideChange{
+	self.journal.append(suicideChange{
 		account:     &addr,
 		prev:        stateObject.suicided,
 		prevbalance: new(big.Int).Set(stateObject.Balance()),
@@ -337,7 +338,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 }
 
 //
-// Setting, updating & deleting state object methods
+// Setting, updating & deleting state object methods.
 //
 
 // updateStateObject writes the given object to the trie.
@@ -379,7 +380,7 @@ func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObje
 		return nil
 	}
 	// Insert into the live set.
-	obj := newObject(self, addr, data, self.MarkStateObjectDirty)
+	obj := newObject(self, addr, data)
 	self.setStateObject(obj)
 	return obj
 }
@@ -388,7 +389,7 @@ func (self *StateDB) setStateObject(object *stateObject) {
 	self.stateObjects[object.Address()] = object
 }
 
-// Retrieve a state object or create a new state object if nil
+// Retrieve a state object or create a new state object if nil.
 func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	stateObject := self.getStateObject(addr)
 	if stateObject == nil || stateObject.deleted {
@@ -397,22 +398,16 @@ func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	return stateObject
 }
 
-// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly
-// state object cache iteration to find a handful of modified ones.
-func (self *StateDB) MarkStateObjectDirty(addr common.Address) {
-	self.stateObjectsDirty[addr] = struct{}{}
-}
-
 // createObject creates a new state object. If there is an existing account with
 // the given address, it is overwritten and returned as the second return value.
 func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = self.getStateObject(addr)
-	newobj = newObject(self, addr, Account{}, self.MarkStateObjectDirty)
+	newobj = newObject(self, addr, Account{})
 	newobj.setNonce(0) // sets the object to dirty
 	if prev == nil {
-		self.journal = append(self.journal, createObjectChange{account: &addr})
+		self.journal.append(createObjectChange{account: &addr})
 	} else {
-		self.journal = append(self.journal, resetObjectChange{prev: prev})
+		self.journal.append(resetObjectChange{prev: prev})
 	}
 	self.setStateObject(newobj)
 	return newobj, prev
@@ -466,18 +461,35 @@ func (self *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                self.db,
 		trie:              self.db.CopyTrie(self.trie),
-		stateObjects:      make(map[common.Address]*stateObject, len(self.stateObjectsDirty)),
-		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
+		stateObjects:      make(map[common.Address]*stateObject, len(self.journal.dirties)),
+		stateObjectsDirty: make(map[common.Address]struct{}, len(self.journal.dirties)),
 		refund:            self.refund,
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}
 	// Copy the dirty states, logs, and preimages
-	for addr := range self.stateObjectsDirty {
-		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty)
-		state.stateObjectsDirty[addr] = struct{}{}
+	for addr := range self.journal.dirties {
+		// As documented [here](https://github.com/ethereum/go-ethereum/pull/16485#issuecomment-380438527),
+		// and in the Finalise-method, there is a case where an object is in the journal but not
+		// in the stateObjects: OOG after touch on ripeMD prior to Byzantium. Thus, we need to check for
+		// nil
+		if object, exist := self.stateObjects[addr]; exist {
+			state.stateObjects[addr] = object.deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
 	}
+	// Above, we don't copy the actual journal. This means that if the copy is copied, the
+	// loop above will be a no-op, since the copy's journal is empty.
+	// Thus, here we iterate over stateObjects, to enable copies of copies
+	for addr := range self.stateObjectsDirty {
+		if _, exist := state.stateObjects[addr]; !exist {
+			state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
+	}
+
 	for hash, logs := range self.logs {
 		state.logs[hash] = make([]*types.Log, len(logs))
 		copy(state.logs[hash], logs)
@@ -492,7 +504,7 @@ func (self *StateDB) Copy() *StateDB {
 func (self *StateDB) Snapshot() int {
 	id := self.nextRevisionId
 	self.nextRevisionId++
-	self.validRevisions = append(self.validRevisions, revision{id, len(self.journal)})
+	self.validRevisions = append(self.validRevisions, revision{id, self.journal.length()})
 	return id
 }
 
@@ -507,13 +519,8 @@ func (self *StateDB) RevertToSnapshot(revid int) {
 	}
 	snapshot := self.validRevisions[idx].journalIndex
 
-	// Replay the journal to undo changes.
-	for i := len(self.journal) - 1; i >= snapshot; i-- {
-		self.journal[i].undo(self)
-	}
-	self.journal = self.journal[:snapshot]
-
-	// Remove invalidated snapshots from the stack.
+	// Replay the journal to undo changes and remove invalidated snapshots
+	self.journal.revert(self, snapshot)
 	self.validRevisions = self.validRevisions[:idx]
 }
 
@@ -525,14 +532,25 @@ func (self *StateDB) GetRefund() uint64 {
 // Finalise finalises the state by removing the self destructed objects
 // and clears the journal as well as the refunds.
 func (s *StateDB) Finalise(deleteEmptyObjects bool) {
-	for addr := range s.stateObjectsDirty {
-		stateObject := s.stateObjects[addr]
+	for addr := range s.journal.dirties {
+		stateObject, exist := s.stateObjects[addr]
+		if !exist {
+			// ripeMD is 'touched' at block 1714175, in tx 0x1237f737031e40bcde4a8b7e717b2d15e3ecadfe49bb1bbc71ee9deb09c6fcf2
+			// That tx goes out of gas, and although the notion of 'touched' does not exist there, the
+			// touch-event will still be recorded in the journal. Since ripeMD is a special snowflake,
+			// it will persist in the journal even though the journal is reverted. In this special circumstance,
+			// it may exist in `s.journal.dirties` but not in `s.stateObjects`.
+			// Thus, we can safely ignore it here
+			continue
+		}
+
 		if stateObject.suicided || (deleteEmptyObjects && stateObject.empty()) {
 			s.deleteStateObject(stateObject)
 		} else {
 			stateObject.updateRoot(s.db)
 			s.updateStateObject(stateObject)
 		}
+		s.stateObjectsDirty[addr] = struct{}{}
 	}
 	// Invalidate journal because reverting across transactions is not allowed.
 	s.clearJournalAndRefund()
@@ -576,7 +594,7 @@ func (s *StateDB) DeleteSuicides() {
 }
 
 func (s *StateDB) clearJournalAndRefund() {
-	s.journal = nil
+	s.journal = newJournal()
 	s.validRevisions = s.validRevisions[:0]
 	s.refund = 0
 }
@@ -585,6 +603,9 @@ func (s *StateDB) clearJournalAndRefund() {
 func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
+	for addr := range s.journal.dirties {
+		s.stateObjectsDirty[addr] = struct{}{}
+	}
 	// Commit objects to the trie.
 	for addr, stateObject := range s.stateObjects {
 		_, isDirty := s.stateObjectsDirty[addr]

core/state/statedb_test.go

@@ -413,11 +413,28 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 
 	snapshot := s.state.Snapshot()
 	s.state.AddBalance(common.Address{}, new(big.Int))
-	if len(s.state.stateObjectsDirty) != 1 {
+
+	if len(s.state.journal.dirties) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
 	s.state.RevertToSnapshot(snapshot)
-	if len(s.state.stateObjectsDirty) != 0 {
+	if len(s.state.journal.dirties) != 0 {
 		c.Fatal("expected no dirty state object")
 	}
 }
+
+// TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy.
+// See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512
+func TestCopyOfCopy(t *testing.T) {
+	db, _ := ethdb.NewMemDatabase()
+	sdb, _ := New(common.Hash{}, NewDatabase(db))
+	addr := common.HexToAddress("aaaa")
+	sdb.SetBalance(addr, big.NewInt(42))
+
+	if got := sdb.Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("1st copy fail, expected 42, got %v", got)
+	}
+	if got := sdb.Copy().Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("2nd copy fail, expected 42, got %v", got)
+	}
+}

core/state_transition.go

@@ -132,28 +132,12 @@ func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, uint64, bool,
 	return NewStateTransition(evm, msg, gp).TransitionDb()
 }
 
-func (st *StateTransition) from() vm.AccountRef {
-	f := st.msg.From()
-	if !st.state.Exist(f) {
-		st.state.CreateAccount(f)
+// to returns the recipient of the message.
+func (st *StateTransition) to() common.Address {
+	if st.msg == nil || st.msg.To() == nil /* contract creation */ {
+		return common.Address{}
 	}
-	return vm.AccountRef(f)
-}
-
-func (st *StateTransition) to() vm.AccountRef {
-	if st.msg == nil {
-		return vm.AccountRef{}
-	}
-	to := st.msg.To()
-	if to == nil {
-		return vm.AccountRef{} // contract creation
-	}
-
-	reference := vm.AccountRef(*to)
-	if !st.state.Exist(*to) {
-		st.state.CreateAccount(*to)
-	}
-	return reference
+	return *st.msg.To()
 }
 
 func (st *StateTransition) useGas(amount uint64) error {
@@ -166,12 +150,8 @@ func (st *StateTransition) useGas(amount uint64) error {
 }
 
 func (st *StateTransition) buyGas() error {
-	var (
-		state  = st.state
-		sender = st.from()
-	)
 	mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.Gas()), st.gasPrice)
-	if state.GetBalance(sender.Address()).Cmp(mgval) < 0 {
+	if st.state.GetBalance(st.msg.From()).Cmp(mgval) < 0 {
 		return errInsufficientBalanceForGas
 	}
 	if err := st.gp.SubGas(st.msg.Gas()); err != nil {
@@ -180,20 +160,17 @@ func (st *StateTransition) buyGas() error {
 	st.gas += st.msg.Gas()
 
 	st.initialGas = st.msg.Gas()
-	state.SubBalance(sender.Address(), mgval)
+	st.state.SubBalance(st.msg.From(), mgval)
 	return nil
 }
 
 func (st *StateTransition) preCheck() error {
-	msg := st.msg
-	sender := st.from()
-
-	// Make sure this transaction's nonce is correct
-	if msg.CheckNonce() {
-		nonce := st.state.GetNonce(sender.Address())
-		if nonce < msg.Nonce() {
+	// Make sure this transaction's nonce is correct.
+	if st.msg.CheckNonce() {
+		nonce := st.state.GetNonce(st.msg.From())
+		if nonce < st.msg.Nonce() {
 			return ErrNonceTooHigh
-		} else if nonce > msg.Nonce() {
+		} else if nonce > st.msg.Nonce() {
 			return ErrNonceTooLow
 		}
 	}
@@ -208,8 +185,7 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		return
 	}
 	msg := st.msg
-	sender := st.from() // err checked in preCheck
-
+	sender := vm.AccountRef(msg.From())
 	homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber)
 	contractCreation := msg.To() == nil
 
@@ -233,8 +209,8 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		ret, _, st.gas, vmerr = evm.Create(sender, st.data, st.gas, st.value)
 	} else {
 		// Increment the nonce for the next transaction
-		st.state.SetNonce(sender.Address(), st.state.GetNonce(sender.Address())+1)
-		ret, st.gas, vmerr = evm.Call(sender, st.to().Address(), st.data, st.gas, st.value)
+		st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1)
+		ret, st.gas, vmerr = evm.Call(sender, st.to(), st.data, st.gas, st.value)
 	}
 	if vmerr != nil {
 		log.Debug("VM returned with error", "err", vmerr)
@@ -260,10 +236,8 @@ func (st *StateTransition) refundGas() {
 	st.gas += refund
 
 	// Return ETH for remaining gas, exchanged at the original rate.
-	sender := st.from()
-
 	remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gas), st.gasPrice)
-	st.state.AddBalance(sender.Address(), remaining)
+	st.state.AddBalance(st.msg.From(), remaining)
 
 	// Also return remaining gas to the block gas counter so it is
 	// available for the next transaction.

core/tx_list.go

@@ -367,9 +367,20 @@ func (l *txList) Flatten() types.Transactions {
 // price-sorted transactions to discard when the pool fills up.
 type priceHeap []*types.Transaction
 
-func (h priceHeap) Len() int           { return len(h) }
-func (h priceHeap) Less(i, j int) bool { return h[i].GasPrice().Cmp(h[j].GasPrice()) < 0 }
-func (h priceHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h priceHeap) Len() int      { return len(h) }
+func (h priceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+
+func (h priceHeap) Less(i, j int) bool {
+	// Sort primarily by price, returning the cheaper one
+	switch h[i].GasPrice().Cmp(h[j].GasPrice()) {
+	case -1:
+		return true
+	case 1:
+		return false
+	}
+	// If the prices match, stabilize via nonces (high nonce is worse)
+	return h[i].Nonce() > h[j].Nonce()
+}
 
 func (h *priceHeap) Push(x interface{}) {
 	*h = append(*h, x.(*types.Transaction))

core/tx_pool.go

@@ -320,7 +320,7 @@ func (pool *TxPool) loop() {
 				// Any non-locals old enough should be removed
 				if time.Since(pool.beats[addr]) > pool.config.Lifetime {
 					for _, tx := range pool.queue[addr].Flatten() {
-						pool.removeTx(tx.Hash())
+						pool.removeTx(tx.Hash(), true)
 					}
 				}
 			}
@@ -468,7 +468,7 @@ func (pool *TxPool) SetGasPrice(price *big.Int) {
 
 	pool.gasPrice = price
 	for _, tx := range pool.priced.Cap(price, pool.locals) {
-		pool.removeTx(tx.Hash())
+		pool.removeTx(tx.Hash(), false)
 	}
 	log.Info("Transaction pool price threshold updated", "price", price)
 }
@@ -630,7 +630,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		for _, tx := range drop {
 			log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice())
 			underpricedTxCounter.Inc(1)
-			pool.removeTx(tx.Hash())
+			pool.removeTx(tx.Hash(), false)
 		}
 	}
 	// If the transaction is replacing an already pending one, do directly
@@ -695,8 +695,10 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er
 		pool.priced.Removed()
 		queuedReplaceCounter.Inc(1)
 	}
-	pool.all[hash] = tx
-	pool.priced.Put(tx)
+	if pool.all[hash] == nil {
+		pool.all[hash] = tx
+		pool.priced.Put(tx)
+	}
 	return old != nil, nil
 }
 
@@ -862,7 +864,7 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction {
 
 // removeTx removes a single transaction from the queue, moving all subsequent
 // transactions back to the future queue.
-func (pool *TxPool) removeTx(hash common.Hash) {
+func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 	// Fetch the transaction we wish to delete
 	tx, ok := pool.all[hash]
 	if !ok {
@@ -872,8 +874,9 @@ func (pool *TxPool) removeTx(hash common.Hash) {
 
 	// Remove it from the list of known transactions
 	delete(pool.all, hash)
-	pool.priced.Removed()
-
+	if outofbound {
+		pool.priced.Removed()
+	}
 	// Remove the transaction from the pending lists and reset the account nonce
 	if pending := pool.pending[addr]; pending != nil {
 		if removed, invalids := pending.Remove(tx); removed {
@@ -1052,7 +1055,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Drop all transactions if they are less than the overflow
 			if size := uint64(list.Len()); size <= drop {
 				for _, tx := range list.Flatten() {
-					pool.removeTx(tx.Hash())
+					pool.removeTx(tx.Hash(), true)
 				}
 				drop -= size
 				queuedRateLimitCounter.Inc(int64(size))
@@ -1061,7 +1064,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Otherwise drop only last few transactions
 			txs := list.Flatten()
 			for i := len(txs) - 1; i >= 0 && drop > 0; i-- {
-				pool.removeTx(txs[i].Hash())
+				pool.removeTx(txs[i].Hash(), true)
 				drop--
 				queuedRateLimitCounter.Inc(1)
 			}

core/tx_pool_test.go

@@ -209,15 +209,10 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
 
 	pool.lockedReset(nil, nil)
 
-	pendingTx, err := pool.Pending()
+	_, err := pool.Pending()
 	if err != nil {
 		t.Fatalf("Could not fetch pending transactions: %v", err)
 	}
-
-	for addr, txs := range pendingTx {
-		t.Logf("%0x: %d\n", addr, len(txs))
-	}
-
 	nonce = pool.State().GetNonce(address)
 	if nonce != 2 {
 		t.Fatalf("Invalid nonce, want 2, got %d", nonce)
@@ -350,7 +345,7 @@ func TestTransactionChainFork(t *testing.T) {
 	if _, err := pool.add(tx, false); err != nil {
 		t.Error("didn't expect error", err)
 	}
-	pool.removeTx(tx.Hash())
+	pool.removeTx(tx.Hash(), true)
 
 	// reset the pool's internal state
 	resetState()
@@ -1388,13 +1383,13 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 		t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
 	// Ensure that adding high priced transactions drops cheap ones, but not own
-	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que -
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
 	pending, queued = pool.Stats()
@@ -1404,7 +1399,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 2 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
 	}
-	if err := validateEvents(events, 2); err != nil {
+	if err := validateEvents(events, 1); err != nil {
 		t.Fatalf("additional event firing failed: %v", err)
 	}
 	if err := validateTxPoolInternals(pool); err != nil {
@@ -1430,6 +1425,74 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	}
 }
 
+// Tests that more expensive transactions push out cheap ones from the pool, but
+// without producing instability by creating gaps that start jumping transactions
+// back and forth between queued/pending.
+func TestTransactionPoolStableUnderpricing(t *testing.T) {
+	t.Parallel()
+
+	// Create the pool to test the pricing enforcement with
+	db, _ := ethdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
+
+	config := testTxPoolConfig
+	config.GlobalSlots = 128
+	config.GlobalQueue = 0
+
+	pool := NewTxPool(config, params.TestChainConfig, blockchain)
+	defer pool.Stop()
+
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
+	// Create a number of test accounts and fund them
+	keys := make([]*ecdsa.PrivateKey, 2)
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
+	}
+	// Fill up the entire queue with the same transaction price points
+	txs := types.Transactions{}
+	for i := uint64(0); i < config.GlobalSlots; i++ {
+		txs = append(txs, pricedTransaction(i, 100000, big.NewInt(1), keys[0]))
+	}
+	pool.AddRemotes(txs)
+
+	pending, queued := pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, int(config.GlobalSlots)); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+	// Ensure that adding high priced transactions drops a cheap, but doesn't produce a gap
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+		t.Fatalf("failed to add well priced transaction: %v", err)
+	}
+	pending, queued = pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("additional event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+}
+
 // Tests that the pool rejects replacement transactions that don't meet the minimum
 // price bump required.
 func TestTransactionReplacement(t *testing.T) {

core/types/block.go

@@ -19,7 +19,6 @@ package types
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 	"math/big"
 	"sort"
@@ -389,40 +388,6 @@ func (b *Block) Hash() common.Hash {
 	return v
 }
 
-func (b *Block) String() string {
-	str := fmt.Sprintf(`Block(#%v): Size: %v {
-MinerHash: %x
-%v
-Transactions:
-%v
-Uncles:
-%v
-}
-`, b.Number(), b.Size(), b.header.HashNoNonce(), b.header, b.transactions, b.uncles)
-	return str
-}
-
-func (h *Header) String() string {
-	return fmt.Sprintf(`Header(%x):
-[
-	ParentHash:	    %x
-	UncleHash:	    %x
-	Coinbase:	    %x
-	Root:		    %x
-	TxSha		    %x
-	ReceiptSha:	    %x
-	Bloom:		    %x
-	Difficulty:	    %v
-	Number:		    %v
-	GasLimit:	    %v
-	GasUsed:	    %v
-	Time:		    %v
-	Extra:		    %s
-	MixDigest:      %x
-	Nonce:		    %x
-]`, h.Hash(), h.ParentHash, h.UncleHash, h.Coinbase, h.Root, h.TxHash, h.ReceiptHash, h.Bloom, h.Difficulty, h.Number, h.GasLimit, h.GasUsed, h.Time, h.Extra, h.MixDigest, h.Nonce)
-}
-
 type Blocks []*Block
 
 type BlockBy func(b1, b2 *Block) bool

core/types/log.go

@@ -17,7 +17,6 @@
 package types
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -95,10 +94,6 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
-func (l *Log) String() string {
-	return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
-}
-
 // LogForStorage is a wrapper around a Log that flattens and parses the entire content of
 // a log including non-consensus fields.
 type LogForStorage Log

core/types/receipt.go

@@ -149,14 +149,6 @@ func (r *Receipt) Size() common.StorageSize {
 	return size
 }
 
-// String implements the Stringer interface.
-func (r *Receipt) String() string {
-	if len(r.PostState) == 0 {
-		return fmt.Sprintf("receipt{status=%d cgas=%v bloom=%x logs=%v}", r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs)
-	}
-	return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
-}
-
 // ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
 // entire content of a receipt, as opposed to only the consensus fields originally.
 type ReceiptForStorage Receipt

core/types/transaction.go

@@ -19,7 +19,6 @@ package types
 import (
 	"container/heap"
 	"errors"
-	"fmt"
 	"io"
 	"math/big"
 	"sync/atomic"
@@ -262,58 +261,6 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
 	return tx.data.V, tx.data.R, tx.data.S
 }
 
-func (tx *Transaction) String() string {
-	var from, to string
-	if tx.data.V != nil {
-		// make a best guess about the signer and use that to derive
-		// the sender.
-		signer := deriveSigner(tx.data.V)
-		if f, err := Sender(signer, tx); err != nil { // derive but don't cache
-			from = "[invalid sender: invalid sig]"
-		} else {
-			from = fmt.Sprintf("%x", f[:])
-		}
-	} else {
-		from = "[invalid sender: nil V field]"
-	}
-
-	if tx.data.Recipient == nil {
-		to = "[contract creation]"
-	} else {
-		to = fmt.Sprintf("%x", tx.data.Recipient[:])
-	}
-	enc, _ := rlp.EncodeToBytes(&tx.data)
-	return fmt.Sprintf(`
-	TX(%x)
-	Contract: %v
-	From:     %s
-	To:       %s
-	Nonce:    %v
-	GasPrice: %#x
-	GasLimit  %#x
-	Value:    %#x
-	Data:     0x%x
-	V:        %#x
-	R:        %#x
-	S:        %#x
-	Hex:      %x
-`,
-		tx.Hash(),
-		tx.data.Recipient == nil,
-		from,
-		to,
-		tx.data.AccountNonce,
-		tx.data.Price,
-		tx.data.GasLimit,
-		tx.data.Amount,
-		tx.data.Payload,
-		tx.data.V,
-		tx.data.R,
-		tx.data.S,
-		enc,
-	)
-}
-
 // Transactions is a Transaction slice type for basic sorting.
 type Transactions []*Transaction
 

core/vm/interpreter.go

@@ -42,7 +42,7 @@ type Config struct {
 }
 
 // Interpreter is used to run Ethereum based contracts and will utilise the
-// passed evmironment to query external sources for state information.
+// passed environment to query external sources for state information.
 // The Interpreter will run the byte code VM based on the passed
 // configuration.
 type Interpreter struct {
@@ -184,7 +184,7 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 			}
 		}
 		// consume the gas and return an error if not enough gas is available.
-		// cost is explicitly set so that the capture state defer method cas get the proper cost
+		// cost is explicitly set so that the capture state defer method can get the proper cost
 		cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize)
 		if err != nil || !contract.UseGas(cost) {
 			return nil, ErrOutOfGas

core/vm/logger.go

@@ -45,6 +45,7 @@ type LogConfig struct {
 	DisableMemory  bool // disable memory capture
 	DisableStack   bool // disable stack capture
 	DisableStorage bool // disable storage capture
+	Debug          bool // print output during capture end
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
@@ -184,6 +185,12 @@ func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost ui
 func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error {
 	l.output = output
 	l.err = err
+	if l.cfg.Debug {
+		fmt.Printf("0x%x\n", output)
+		if err != nil {
+			fmt.Printf(" error: %v\n", err)
+		}
+	}
 	return nil
 }
 

core/vm/runtime/runtime.go

@@ -103,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db))
 	}
 	var (
-		address = common.StringToAddress("contract")
+		address = common.BytesToAddress([]byte("contract"))
 		vmenv   = NewEnv(cfg)
 		sender  = vm.AccountRef(cfg.Origin)
 	)
@@ -113,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 	// Call the code with the given configuration.
 	ret, _, err := vmenv.Call(
 		sender,
-		common.StringToAddress("contract"),
+		common.BytesToAddress([]byte("contract")),
 		input,
 		cfg.GasLimit,
 		cfg.Value,

crypto/secp256k1/curve.go

@@ -290,11 +290,11 @@ func init() {
 	// See SEC 2 section 2.7.1
 	// curve parameters taken from:
 	// http://www.secg.org/collateral/sec2_final.pdf
-	theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
-	theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
-	theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
-	theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
-	theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
+	theCurve.P = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F")
+	theCurve.N = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141")
+	theCurve.B = math.MustParseBig256("0x0000000000000000000000000000000000000000000000000000000000000007")
+	theCurve.Gx = math.MustParseBig256("0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798")
+	theCurve.Gy = math.MustParseBig256("0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8")
 	theCurve.BitSize = 256
 }
 

eth/backend.go

@@ -63,7 +63,7 @@ type Ethereum struct {
 	chainConfig *params.ChainConfig
 
 	// Channel for shutting down the service
-	shutdownChan  chan bool    // Channel for shutting down the ethereum
+	shutdownChan  chan bool    // Channel for shutting down the Ethereum
 	stopDbUpgrade func() error // stop chain db sequential key upgrade
 
 	// Handlers
@@ -351,7 +351,7 @@ func (s *Ethereum) StartMining(local bool) error {
 	if local {
 		// If local (CPU) mining is started, we can disable the transaction rejection
 		// mechanism introduced to speed sync times. CPU mining on mainnet is ludicrous
-		// so noone will ever hit this path, whereas marking sync done on CPU mining
+		// so none will ever hit this path, whereas marking sync done on CPU mining
 		// will ensure that private networks work in single miner mode too.
 		atomic.StoreUint32(&s.protocolManager.acceptTxs, 1)
 	}

eth/db_upgrade.go

@@ -62,7 +62,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 			failed    error
 		)
 		for failed == nil && it.Next() {
-			// Skip any entries that don't look like old transaction meta entires (<hash>0x01)
+			// Skip any entries that don't look like old transaction meta entries (<hash>0x01)
 			key := it.Key()
 			if len(key) != common.HashLength+1 || key[common.HashLength] != 0x01 {
 				continue
@@ -86,7 +86,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 				}
 			}
 			// Convert the old metadata to a new lookup entry, delete duplicate data
-			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new looku entry
+			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new lookup entry
 				if failed = db.Delete(hash); failed == nil { // Delete the duplicate transaction data
 					if failed = db.Delete(append([]byte("receipts-"), hash...)); failed == nil { // Delete the duplicate receipt data
 						if failed = db.Delete(key); failed != nil { // Delete the old transaction metadata

eth/downloader/downloader.go

@@ -47,7 +47,7 @@ var (
 
 	MaxForkAncestry  = 3 * params.EpochDuration // Maximum chain reorganisation
 	rttMinEstimate   = 2 * time.Second          // Minimum round-trip time to target for download requests
-	rttMaxEstimate   = 20 * time.Second         // Maximum rount-trip time to target for download requests
+	rttMaxEstimate   = 20 * time.Second         // Maximum round-trip time to target for download requests
 	rttMinConfidence = 0.1                      // Worse confidence factor in our estimated RTT value
 	ttlScaling       = 3                        // Constant scaling factor for RTT -> TTL conversion
 	ttlLimit         = time.Minute              // Maximum TTL allowance to prevent reaching crazy timeouts
@@ -135,9 +135,10 @@ type Downloader struct {
 	stateCh        chan dataPack // [eth/63] Channel receiving inbound node state data
 
 	// Cancellation and termination
-	cancelPeer string        // Identifier of the peer currently being used as the master (cancel on drop)
-	cancelCh   chan struct{} // Channel to cancel mid-flight syncs
-	cancelLock sync.RWMutex  // Lock to protect the cancel channel and peer in delivers
+	cancelPeer string         // Identifier of the peer currently being used as the master (cancel on drop)
+	cancelCh   chan struct{}  // Channel to cancel mid-flight syncs
+	cancelLock sync.RWMutex   // Lock to protect the cancel channel and peer in delivers
+	cancelWg   sync.WaitGroup // Make sure all fetcher goroutines have exited.
 
 	quitCh   chan struct{} // Quit channel to signal termination
 	quitLock sync.RWMutex  // Lock to prevent double closes
@@ -476,12 +477,11 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 // spawnSync runs d.process and all given fetcher functions to completion in
 // separate goroutines, returning the first error that appears.
 func (d *Downloader) spawnSync(fetchers []func() error) error {
-	var wg sync.WaitGroup
 	errc := make(chan error, len(fetchers))
-	wg.Add(len(fetchers))
+	d.cancelWg.Add(len(fetchers))
 	for _, fn := range fetchers {
 		fn := fn
-		go func() { defer wg.Done(); errc <- fn() }()
+		go func() { defer d.cancelWg.Done(); errc <- fn() }()
 	}
 	// Wait for the first error, then terminate the others.
 	var err error
@@ -498,12 +498,10 @@ func (d *Downloader) spawnSync(fetchers []func() error) error {
 	}
 	d.queue.Close()
 	d.Cancel()
-	wg.Wait()
 	return err
 }
 
-// Cancel cancels all of the operations and resets the queue. It returns true
-// if the cancel operation was completed.
+// Cancel cancels all of the operations and resets the queue.
 func (d *Downloader) Cancel() {
 	// Close the current cancel channel
 	d.cancelLock.Lock()
@@ -516,6 +514,7 @@ func (d *Downloader) Cancel() {
 		}
 	}
 	d.cancelLock.Unlock()
+	d.cancelWg.Wait()
 }
 
 // Terminate interrupts the downloader, canceling all pending operations.
@@ -884,7 +883,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64)
 // immediately to the header processor to keep the rest of the pipeline full even
 // in the case of header stalls.
 //
-// The method returs the entire filled skeleton and also the number of headers
+// The method returns the entire filled skeleton and also the number of headers
 // already forwarded for processing.
 func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ([]*types.Header, int, error) {
 	log.Debug("Filling up skeleton", "from", from)
@@ -1377,7 +1376,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 		pivot = height - uint64(fsMinFullBlocks)
 	}
 	// To cater for moving pivot points, track the pivot block and subsequently
-	// accumulated download results separatey.
+	// accumulated download results separately.
 	var (
 		oldPivot *fetchResult   // Locked in pivot block, might change eventually
 		oldTail  []*fetchResult // Downloaded content after the pivot
@@ -1615,7 +1614,7 @@ func (d *Downloader) qosReduceConfidence() {
 //
 // Note, the returned RTT is .9 of the actually estimated RTT. The reason is that
 // the downloader tries to adapt queries to the RTT, so multiple RTT values can
-// be adapted to, but smaller ones are preffered (stabler download stream).
+// be adapted to, but smaller ones are preferred (stabler download stream).
 func (d *Downloader) requestRTT() time.Duration {
 	return time.Duration(atomic.LoadUint64(&d.rttEstimate)) * 9 / 10
 }

eth/downloader/downloader_test.go

@@ -159,7 +159,7 @@ func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentRec
 	// Create the common suffix
 	hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false)
 
-	// Create the forks, making the second heavyer if non balanced forks were requested
+	// Create the forks, making the second heavier if non balanced forks were requested
 	hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false)
 	hashes1 = append(hashes1, hashes[1:]...)
 

eth/downloader/fakepeer.go

@@ -27,7 +27,7 @@ import (
 
 // FakePeer is a mock downloader peer that operates on a local database instance
 // instead of being an actual live node. It's useful for testing and to implement
-// sync commands from an xisting local database.
+// sync commands from an existing local database.
 type FakePeer struct {
 	id string
 	db ethdb.Database
@@ -48,7 +48,7 @@ func (p *FakePeer) Head() (common.Hash, *big.Int) {
 }
 
 // RequestHeadersByHash implements downloader.Peer, returning a batch of headers
-// defined by the origin hash and the associaed query parameters.
+// defined by the origin hash and the associated query parameters.
 func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header
@@ -92,7 +92,7 @@ func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int,
 }
 
 // RequestHeadersByNumber implements downloader.Peer, returning a batch of headers
-// defined by the origin number and the associaed query parameters.
+// defined by the origin number and the associated query parameters.
 func (p *FakePeer) RequestHeadersByNumber(number uint64, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header

eth/downloader/peer.go

@@ -551,7 +551,7 @@ func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peerC
 // medianRTT returns the median RTT of the peerset, considering only the tuning
 // peers if there are more peers available.
 func (ps *peerSet) medianRTT() time.Duration {
-	// Gather all the currnetly measured round trip times
+	// Gather all the currently measured round trip times
 	ps.lock.RLock()
 	defer ps.lock.RUnlock()
 

eth/downloader/queue.go

@@ -275,7 +275,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
 	if q.headerResults != nil {
 		panic("skeleton assembly already in progress")
 	}
-	// Shedule all the header retrieval tasks for the skeleton assembly
+	// Schedule all the header retrieval tasks for the skeleton assembly
 	q.headerTaskPool = make(map[uint64]*types.Header)
 	q.headerTaskQueue = prque.New()
 	q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains

eth/downloader/statesync.go

@@ -31,7 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-// stateReq represents a batch of state fetch requests groupped together into
+// stateReq represents a batch of state fetch requests grouped together into
 // a single data retrieval network packet.
 type stateReq struct {
 	items    []common.Hash              // Hashes of the state items to download
@@ -139,7 +139,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 
 		// Handle incoming state packs:
 		case pack := <-d.stateCh:
-			// Discard any data not requested (or previsouly timed out)
+			// Discard any data not requested (or previously timed out)
 			req := active[pack.PeerId()]
 			if req == nil {
 				log.Debug("Unrequested node data", "peer", pack.PeerId(), "len", pack.Items())
@@ -182,7 +182,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 		case req := <-d.trackStateReq:
 			// If an active request already exists for this peer, we have a problem. In
 			// theory the trie node schedule must never assign two requests to the same
-			// peer. In practive however, a peer might receive a request, disconnect and
+			// peer. In practice however, a peer might receive a request, disconnect and
 			// immediately reconnect before the previous times out. In this case the first
 			// request is never honored, alas we must not silently overwrite it, as that
 			// causes valid requests to go missing and sync to get stuck.
@@ -228,7 +228,7 @@ type stateSync struct {
 	err        error          // Any error hit during sync (set before completion)
 }
 
-// stateTask represents a single trie node download taks, containing a set of
+// stateTask represents a single trie node download task, containing a set of
 // peers already attempted retrieval from to detect stalled syncs and abort.
 type stateTask struct {
 	attempts map[string]struct{}
@@ -274,15 +274,21 @@ func (s *stateSync) Cancel() error {
 // receive data from peers, rather those are buffered up in the downloader and
 // pushed here async. The reason is to decouple processing from data receipt
 // and timeouts.
-func (s *stateSync) loop() error {
+func (s *stateSync) loop() (err error) {
 	// Listen for new peer events to assign tasks to them
 	newPeer := make(chan *peerConnection, 1024)
 	peerSub := s.d.peers.SubscribeNewPeers(newPeer)
 	defer peerSub.Unsubscribe()
+	defer func() {
+		cerr := s.commit(true)
+		if err == nil {
+			err = cerr
+		}
+	}()
 
 	// Keep assigning new tasks until the sync completes or aborts
 	for s.sched.Pending() > 0 {
-		if err := s.commit(false); err != nil {
+		if err = s.commit(false); err != nil {
 			return err
 		}
 		s.assignTasks()
@@ -307,14 +313,14 @@ func (s *stateSync) loop() error {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err := s.process(req); err != nil {
+			if err = s.process(req); err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
 			req.peer.SetNodeDataIdle(len(req.response))
 		}
 	}
-	return s.commit(true)
+	return nil
 }
 
 func (s *stateSync) commit(force bool) error {
@@ -323,7 +329,9 @@ func (s *stateSync) commit(force bool) error {
 	}
 	start := time.Now()
 	b := s.d.stateDB.NewBatch()
-	s.sched.Commit(b)
+	if written, err := s.sched.Commit(b); written == 0 || err != nil {
+		return err
+	}
 	if err := b.Write(); err != nil {
 		return fmt.Errorf("DB write error: %v", err)
 	}
@@ -333,7 +341,7 @@ func (s *stateSync) commit(force bool) error {
 	return nil
 }
 
-// assignTasks attempts to assing new tasks to all idle peers, either from the
+// assignTasks attempts to assign new tasks to all idle peers, either from the
 // batch currently being retried, or fetching new data from the trie sync itself.
 func (s *stateSync) assignTasks() {
 	// Iterate over all idle peers and try to assign them state fetches

eth/fetcher/fetcher.go

@@ -127,7 +127,7 @@ type Fetcher struct {
 	// Block cache
 	queue  *prque.Prque            // Queue containing the import operations (block number sorted)
 	queues map[string]int          // Per peer block counts to prevent memory exhaustion
-	queued map[common.Hash]*inject // Set of already queued blocks (to dedup imports)
+	queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports)
 
 	// Callbacks
 	getBlock       blockRetrievalFn   // Retrieves a block from the local chain

eth/filters/api.go

@@ -98,7 +98,7 @@ func (api *PublicFilterAPI) timeoutLoop() {
 // NewPendingTransactionFilter creates a filter that fetches pending transaction hashes
 // as transactions enter the pending state.
 //
-// It is part of the filter package because this filter can be used throug the
+// It is part of the filter package because this filter can be used through the
 // `eth_getFilterChanges` polling method that is also used for log filters.
 //
 // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter

eth/filters/api_test.go

@@ -29,8 +29,8 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) {
 	var (
 		fromBlock rpc.BlockNumber = 0x123435
 		toBlock   rpc.BlockNumber = 0xabcdef
-		address0                  = common.StringToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
-		address1                  = common.StringToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
+		address0                  = common.HexToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
+		address1                  = common.HexToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
 		topic0                    = common.HexToHash("3ac225168df54212a25c1c01fd35bebfea408fdac2e31ddd6f80a4bbf9a5f1ca")
 		topic1                    = common.HexToHash("9084a792d2f8b16a62b882fd56f7860c07bf5fa91dd8a2ae7e809e5180fef0b3")
 		topic2                    = common.HexToHash("6ccae1c4af4152f460ff510e573399795dfab5dcf1fa60d1f33ac8fdc1e480ce")

eth/handler.go

@@ -96,8 +96,8 @@ type ProtocolManager struct {
 	wg sync.WaitGroup
 }
 
-// NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
-// with the ethereum network.
+// NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
+// with the Ethereum network.
 func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) {
 	// Create the protocol manager with the base fields
 	manager := &ProtocolManager{
@@ -498,20 +498,20 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			return errResp(ErrDecode, "msg %v: %v", msg, err)
 		}
 		// Deliver them all to the downloader for queuing
-		trasactions := make([][]*types.Transaction, len(request))
+		transactions := make([][]*types.Transaction, len(request))
 		uncles := make([][]*types.Header, len(request))
 
 		for i, body := range request {
-			trasactions[i] = body.Transactions
+			transactions[i] = body.Transactions
 			uncles[i] = body.Uncles
 		}
 		// Filter out any explicitly requested bodies, deliver the rest to the downloader
-		filter := len(trasactions) > 0 || len(uncles) > 0
+		filter := len(transactions) > 0 || len(uncles) > 0
 		if filter {
-			trasactions, uncles = pm.fetcher.FilterBodies(p.id, trasactions, uncles, time.Now())
+			transactions, uncles = pm.fetcher.FilterBodies(p.id, transactions, uncles, time.Now())
 		}
-		if len(trasactions) > 0 || len(uncles) > 0 || !filter {
-			err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
+		if len(transactions) > 0 || len(uncles) > 0 || !filter {
+			err := pm.downloader.DeliverBodies(p.id, transactions, uncles)
 			if err != nil {
 				log.Debug("Failed to deliver bodies", "err", err)
 			}

ethclient/ethclient.go

@@ -296,7 +296,7 @@ func (ec *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, err
 // SubscribeNewHead subscribes to notifications about the current blockchain head
 // on the given channel.
 func (ec *Client) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "newHeads", map[string]struct{}{})
+	return ec.c.EthSubscribe(ctx, ch, "newHeads")
 }
 
 // State Access

ethdb/database.go

@@ -17,6 +17,7 @@
 package ethdb
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,17 +33,25 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+const (
+	writeDelayNThreshold       = 200
+	writeDelayThreshold        = 350 * time.Millisecond
+	writeDelayWarningThrottler = 1 * time.Minute
+)
+
 var OpenFileLimit = 64
 
 type LDBDatabase struct {
 	fn string      // filename for reporting
 	db *leveldb.DB // LevelDB instance
 
-	compTimeMeter  metrics.Meter // Meter for measuring the total time spent in database compaction
-	compReadMeter  metrics.Meter // Meter for measuring the data read during compaction
-	compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
-	diskReadMeter  metrics.Meter // Meter for measuring the effective amount of data read
-	diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
+	compTimeMeter    metrics.Meter // Meter for measuring the total time spent in database compaction
+	compReadMeter    metrics.Meter // Meter for measuring the data read during compaction
+	compWriteMeter   metrics.Meter // Meter for measuring the data written during compaction
+	writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
+	writeDelayMeter  metrics.Meter // Meter for measuring the write delay duration due to database compaction
+	diskReadMeter    metrics.Meter // Meter for measuring the effective amount of data read
+	diskWriteMeter   metrics.Meter // Meter for measuring the effective amount of data written
 
 	quitLock sync.Mutex      // Mutex protecting the quit channel access
 	quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
@@ -91,9 +100,6 @@ func (db *LDBDatabase) Path() string {
 
 // Put puts the given key / value to the queue
 func (db *LDBDatabase) Put(key []byte, value []byte) error {
-	// Generate the data to write to disk, update the meter and write
-	//value = rle.Compress(value)
-
 	return db.db.Put(key, value, nil)
 }
 
@@ -103,18 +109,15 @@ func (db *LDBDatabase) Has(key []byte) (bool, error) {
 
 // Get returns the given key if it's present.
 func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
-	// Retrieve the key and increment the miss counter if not found
 	dat, err := db.db.Get(key, nil)
 	if err != nil {
 		return nil, err
 	}
 	return dat, nil
-	//return rle.Decompress(dat)
 }
 
 // Delete deletes the key from the queue and database
 func (db *LDBDatabase) Delete(key []byte) error {
-	// Execute the actual operation
 	return db.db.Delete(key, nil)
 }
 
@@ -153,16 +156,17 @@ func (db *LDBDatabase) LDB() *leveldb.DB {
 
 // Meter configures the database metrics collectors and
 func (db *LDBDatabase) Meter(prefix string) {
-	// Short circuit metering if the metrics system is disabled
-	if !metrics.Enabled {
-		return
+	if metrics.Enabled {
+		// Initialize all the metrics collector at the requested prefix
+		db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
+		db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
+		db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
+		db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil)
+		db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil)
 	}
-	// Initialize all the metrics collector at the requested prefix
-	db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
-	db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
-	db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
-	db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil)
-	db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil)
+	// Initialize write delay metrics no matter we are in metric mode or not.
+	db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil)
+	db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil)
 
 	// Create a quit channel for the periodic collector and run it
 	db.quitLock.Lock()
@@ -184,6 +188,9 @@ func (db *LDBDatabase) Meter(prefix string) {
 //      2   |        523 |    1000.37159 |       7.26059 |      66.86342 |      66.77884
 //      3   |        570 |    1113.18458 |       0.00000 |       0.00000 |       0.00000
 //
+// This is how the write delay look like (currently):
+// DelayN:5 Delay:406.604657ms
+//
 // This is how the iostats look like (currently):
 // Read(MB):3895.04860 Write(MB):3654.64712
 func (db *LDBDatabase) meter(refresh time.Duration) {
@@ -194,6 +201,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 	}
 	// Create storage for iostats.
 	var iostats [2]float64
+
+	// Create storage and warning log tracer for write delay.
+	var (
+		delaystats      [2]int64
+		lastWriteDelay  time.Time
+		lastWriteDelayN time.Time
+	)
+
 	// Iterate ad infinitum and collect the stats
 	for i := 1; ; i++ {
 		// Retrieve the database stats
@@ -242,6 +257,52 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 			db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024))
 		}
 
+		// Retrieve the write delay statistic
+		writedelay, err := db.db.GetProperty("leveldb.writedelay")
+		if err != nil {
+			db.log.Error("Failed to read database write delay statistic", "err", err)
+			return
+		}
+		var (
+			delayN        int64
+			delayDuration string
+			duration      time.Duration
+		)
+		if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s", &delayN, &delayDuration); n != 2 || err != nil {
+			db.log.Error("Write delay statistic not found")
+			return
+		}
+		duration, err = time.ParseDuration(delayDuration)
+		if err != nil {
+			db.log.Error("Failed to parse delay duration", "err", err)
+			return
+		}
+		if db.writeDelayNMeter != nil {
+			db.writeDelayNMeter.Mark(delayN - delaystats[0])
+			// If the write delay number been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			// If a warning that db performance is laggy has been displayed,
+			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
+			if int(db.writeDelayNMeter.Rate1()) > writeDelayNThreshold &&
+				time.Now().After(lastWriteDelayN.Add(writeDelayWarningThrottler)) {
+				db.log.Warn("Write delay number exceeds the threshold (200 per second) in the last minute")
+				lastWriteDelayN = time.Now()
+			}
+		}
+		if db.writeDelayMeter != nil {
+			db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1])
+			// If the write delay duration been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			// If a warning that db performance is laggy has been displayed,
+			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
+			if int64(db.writeDelayMeter.Rate1()) > writeDelayThreshold.Nanoseconds() &&
+				time.Now().After(lastWriteDelay.Add(writeDelayWarningThrottler)) {
+				db.log.Warn("Write delay duration exceeds the threshold (35% of the time) in the last minute")
+				lastWriteDelay = time.Now()
+			}
+		}
+		delaystats[0], delaystats[1] = delayN, duration.Nanoseconds()
+
 		// Retrieve the database iostats.
 		ioStats, err := db.db.GetProperty("leveldb.iostats")
 		if err != nil {

internal/ethapi/api.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -1388,7 +1389,7 @@ func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (strin
 	if block == nil {
 		return "", fmt.Errorf("block #%d not found", number)
 	}
-	return block.String(), nil
+	return spew.Sdump(block), nil
 }
 
 // SeedHash retrieves the seed hash of a block.

les/peer.go

@@ -545,9 +545,11 @@ func (ps *peerSet) notify(n peerSetNotify) {
 func (ps *peerSet) Register(p *peer) error {
 	ps.lock.Lock()
 	if ps.closed {
+		ps.lock.Unlock()
 		return errClosed
 	}
 	if _, ok := ps.peers[p.id]; ok {
+		ps.lock.Unlock()
 		return errAlreadyRegistered
 	}
 	ps.peers[p.id] = p

light/postprocess.go

@@ -58,18 +58,18 @@ type trustedCheckpoint struct {
 var (
 	mainnetCheckpoint = trustedCheckpoint{
 		name:          "mainnet",
-		sectionIdx:    161,
-		sectionHead:   common.HexToHash("75b0c4baa7a62cece48abdcb03b6f31601961c9bece67dcd61df87aad4fc0d8d"),
-		chtRoot:       common.HexToHash("bbbfaa67b29716348997ec21a39c03b8d1fb973f6a43740b865595ba26ee812f"),
-		bloomTrieRoot: common.HexToHash("d6db6e6248354d7453391ce97830072a28ea4216be0bd95a5db9f53b1a64677b"),
+		sectionIdx:    165,
+		sectionHead:   common.HexToHash("21028acf9cd9ce80257221adc437c3c58ce046c4d43c21c3e9b1d1349059ec73"),
+		chtRoot:       common.HexToHash("26b2458cb7d0080d3a39311c914be92c368777a65ec074e1893b8bdc79e3910a"),
+		bloomTrieRoot: common.HexToHash("5d06908769179186165a72db7fc3473b25c28ed27efe78a392a9ff2c3fa67f84"),
 	}
 
 	ropstenCheckpoint = trustedCheckpoint{
 		name:          "ropsten",
-		sectionIdx:    83,
-		sectionHead:   common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"),
-		chtRoot:       common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"),
-		bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"),
+		sectionIdx:    92,
+		sectionHead:   common.HexToHash("21a158f9cc643da13a237cafceb37381072649f7278cf98c5820bfbced7cfcec"),
+		chtRoot:       common.HexToHash("1a8ddb8b086d7a33ca90eea90730225948fa504ae0283b15aff3c15c0e089bf9"),
+		bloomTrieRoot: common.HexToHash("fd192f92afbcdd0020c81ca0625116b5995509659653b10123bd986fe5129cc1"),
 	}
 )
 

miner/worker.go

@@ -117,6 +117,10 @@ type worker struct {
 	currentMu sync.Mutex
 	current   *Work
 
+	snapshotMu    sync.RWMutex
+	snapshotBlock *types.Block
+	snapshotState *state.StateDB
+
 	uncleMu        sync.Mutex
 	possibleUncles map[common.Hash]*types.Block
 
@@ -171,32 +175,28 @@ func (self *worker) setExtra(extra []byte) {
 }
 
 func (self *worker) pending() (*types.Block, *state.StateDB) {
+	if atomic.LoadInt32(&self.mining) == 0 {
+		// return a snapshot to avoid contention on currentMu mutex
+		self.snapshotMu.RLock()
+		defer self.snapshotMu.RUnlock()
+		return self.snapshotBlock, self.snapshotState.Copy()
+	}
+
 	self.currentMu.Lock()
 	defer self.currentMu.Unlock()
-
-	if atomic.LoadInt32(&self.mining) == 0 {
-		return types.NewBlock(
-			self.current.header,
-			self.current.txs,
-			nil,
-			self.current.receipts,
-		), self.current.state.Copy()
-	}
 	return self.current.Block, self.current.state.Copy()
 }
 
 func (self *worker) pendingBlock() *types.Block {
+	if atomic.LoadInt32(&self.mining) == 0 {
+		// return a snapshot to avoid contention on currentMu mutex
+		self.snapshotMu.RLock()
+		defer self.snapshotMu.RUnlock()
+		return self.snapshotBlock
+	}
+
 	self.currentMu.Lock()
 	defer self.currentMu.Unlock()
-
-	if atomic.LoadInt32(&self.mining) == 0 {
-		return types.NewBlock(
-			self.current.header,
-			self.current.txs,
-			nil,
-			self.current.receipts,
-		)
-	}
 	return self.current.Block
 }
 
@@ -268,6 +268,7 @@ func (self *worker) update() {
 				txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs)
 
 				self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase)
+				self.updateSnapshot()
 				self.currentMu.Unlock()
 			} else {
 				// If we're mining, but nothing is being processed, wake on new transactions
@@ -489,6 +490,7 @@ func (self *worker) commitNewWork() {
 		self.unconfirmed.Shift(work.Block.NumberU64() - 1)
 	}
 	self.push(work)
+	self.updateSnapshot()
 }
 
 func (self *worker) commitUncle(work *Work, uncle *types.Header) error {
@@ -506,6 +508,19 @@ func (self *worker) commitUncle(work *Work, uncle *types.Header) error {
 	return nil
 }
 
+func (self *worker) updateSnapshot() {
+	self.snapshotMu.Lock()
+	defer self.snapshotMu.Unlock()
+
+	self.snapshotBlock = types.NewBlock(
+		self.current.header,
+		self.current.txs,
+		nil,
+		self.current.receipts,
+	)
+	self.snapshotState = self.current.state.Copy()
+}
+
 func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) {
 	gp := new(core.GasPool).AddGas(env.header.GasLimit)
 

mobile/types.go

@@ -97,12 +97,6 @@ func (h *Header) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the header for debugging purposes.
-func (h *Header) String() string {
-	return h.header.String()
-}
-
 func (h *Header) GetParentHash() *Hash   { return &Hash{h.header.ParentHash} }
 func (h *Header) GetUncleHash() *Hash    { return &Hash{h.header.UncleHash} }
 func (h *Header) GetCoinbase() *Address  { return &Address{h.header.Coinbase} }
@@ -174,12 +168,6 @@ func (b *Block) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the block for debugging purposes.
-func (b *Block) String() string {
-	return b.block.String()
-}
-
 func (b *Block) GetParentHash() *Hash   { return &Hash{b.block.ParentHash()} }
 func (b *Block) GetUncleHash() *Hash    { return &Hash{b.block.UncleHash()} }
 func (b *Block) GetCoinbase() *Address  { return &Address{b.block.Coinbase()} }
@@ -249,12 +237,6 @@ func (tx *Transaction) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction for debugging purposes.
-func (tx *Transaction) String() string {
-	return tx.tx.String()
-}
-
 func (tx *Transaction) GetData() []byte      { return tx.tx.Data() }
 func (tx *Transaction) GetGas() int64        { return int64(tx.tx.Gas()) }
 func (tx *Transaction) GetGasPrice() *BigInt { return &BigInt{tx.tx.GasPrice()} }
@@ -347,12 +329,6 @@ func (r *Receipt) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction receipt for debugging purposes.
-func (r *Receipt) String() string {
-	return r.receipt.String()
-}
-
 func (r *Receipt) GetPostState() []byte         { return r.receipt.PostState }
 func (r *Receipt) GetCumulativeGasUsed() int64  { return int64(r.receipt.CumulativeGasUsed) }
 func (r *Receipt) GetBloom() *Bloom             { return &Bloom{r.receipt.Bloom} }

node/node.go

@@ -306,47 +306,23 @@ func (n *Node) startIPC(apis []rpc.API) error {
 	// Short circuit if the IPC endpoint isn't being exposed
 	if n.ipcEndpoint == "" {
 		return nil
+
 	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-			return err
-		}
-		n.log.Debug("IPC registered", "service", api.Service, "namespace", api.Namespace)
+	isClosed := func() bool {
+		n.lock.RLock()
+		defer n.lock.RUnlock()
+		return n.ipcListener == nil
 	}
-	// All APIs registered, start the IPC listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = rpc.CreateIPCListener(n.ipcEndpoint); err != nil {
+
+	listener, handler, err := rpc.StartIPCEndpoint(isClosed, n.ipcEndpoint, apis)
+	if err != nil {
 		return err
 	}
-	go func() {
-		n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint)
 
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				// Terminate if the listener was closed
-				n.lock.RLock()
-				closed := n.ipcListener == nil
-				n.lock.RUnlock()
-				if closed {
-					return
-				}
-				// Not closed, just some error; report and continue
-				n.log.Error("IPC accept failed", "err", err)
-				continue
-			}
-			go handler.ServeCodec(rpc.NewJSONCodec(conn), rpc.OptionMethodInvocation|rpc.OptionSubscriptions)
-		}
-	}()
 	// All listeners booted successfully
 	n.ipcListener = listener
 	n.ipcHandler = handler
-
+	n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint)
 	return nil
 }
 
@@ -370,30 +346,10 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 	if endpoint == "" {
 		return nil
 	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return err
-			}
-			n.log.Debug("HTTP registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
+	listener, handler, err := rpc.StartHTTPEndpoint(endpoint, apis, modules, cors, vhosts)
+	if err != nil {
 		return err
 	}
-	go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener)
 	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ","))
 	// All listeners booted successfully
 	n.httpEndpoint = endpoint
@@ -423,32 +379,11 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig
 	if endpoint == "" {
 		return nil
 	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return err
-			}
-			n.log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
+	listener, handler, err := rpc.StartWSEndpoint(endpoint, apis, modules, wsOrigins, exposeAll)
+	if err != nil {
 		return err
 	}
-	go rpc.NewWSServer(wsOrigins, handler).Serve(listener)
 	n.log.Info("WebSocket endpoint opened", "url", fmt.Sprintf("ws://%s", listener.Addr()))
-
 	// All listeners booted successfully
 	n.wsEndpoint = endpoint
 	n.wsListener = listener

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1        // Major version component of the current release
 	VersionMinor = 8        // Minor version component of the current release
-	VersionPatch = 3        // Patch version component of the current release
+	VersionPatch = 4        // Patch version component of the current release
 	VersionMeta  = "stable" // Version metadata to append to the version string
 )
 

rpc/client.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	"os"
 )
 
 var (
@@ -171,6 +172,8 @@ func DialContext(ctx context.Context, rawurl string) (*Client, error) {
 		return DialHTTP(rawurl)
 	case "ws", "wss":
 		return DialWebsocket(ctx, rawurl, "")
+	case "stdio":
+		return DialStdIO(ctx)
 	case "":
 		return DialIPC(ctx, rawurl)
 	default:
@@ -178,13 +181,51 @@ func DialContext(ctx context.Context, rawurl string) (*Client, error) {
 	}
 }
 
+type StdIOConn struct{}
+
+func (io StdIOConn) Read(b []byte) (n int, err error) {
+	return os.Stdin.Read(b)
+}
+
+func (io StdIOConn) Write(b []byte) (n int, err error) {
+	return os.Stdout.Write(b)
+}
+
+func (io StdIOConn) Close() error {
+	return nil
+}
+
+func (io StdIOConn) LocalAddr() net.Addr {
+	return &net.UnixAddr{Name: "stdio", Net: "stdio"}
+}
+
+func (io StdIOConn) RemoteAddr() net.Addr {
+	return &net.UnixAddr{Name: "stdio", Net: "stdio"}
+}
+
+func (io StdIOConn) SetDeadline(t time.Time) error {
+	return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
+}
+
+func (io StdIOConn) SetReadDeadline(t time.Time) error {
+	return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
+}
+
+func (io StdIOConn) SetWriteDeadline(t time.Time) error {
+	return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
+}
+func DialStdIO(ctx context.Context) (*Client, error) {
+	return newClient(ctx, func(_ context.Context) (net.Conn, error) {
+		return StdIOConn{}, nil
+	})
+}
+
 func newClient(initctx context.Context, connectFunc func(context.Context) (net.Conn, error)) (*Client, error) {
 	conn, err := connectFunc(initctx)
 	if err != nil {
 		return nil, err
 	}
 	_, isHTTP := conn.(*httpConn)
-
 	c := &Client{
 		writeConn:   conn,
 		isHTTP:      isHTTP,
@@ -524,13 +565,13 @@ func (c *Client) dispatch(conn net.Conn) {
 			}
 
 		case err := <-c.readErr:
-			log.Debug(fmt.Sprintf("<-readErr: %v", err))
+			log.Debug("<-readErr", "err", err)
 			c.closeRequestOps(err)
 			conn.Close()
 			reading = false
 
 		case newconn := <-c.reconnected:
-			log.Debug(fmt.Sprintf("<-reconnected: (reading=%t) %v", reading, conn.RemoteAddr()))
+			log.Debug("<-reconnected", "reading", reading, "remote", conn.RemoteAddr())
 			if reading {
 				// Wait for the previous read loop to exit. This is a rare case.
 				conn.Close()
@@ -587,7 +628,7 @@ func (c *Client) closeRequestOps(err error) {
 
 func (c *Client) handleNotification(msg *jsonrpcMessage) {
 	if !strings.HasSuffix(msg.Method, notificationMethodSuffix) {
-		log.Debug(fmt.Sprint("dropping non-subscription message: ", msg))
+		log.Debug("dropping non-subscription message", "msg", msg)
 		return
 	}
 	var subResult struct {
@@ -595,7 +636,7 @@ func (c *Client) handleNotification(msg *jsonrpcMessage) {
 		Result json.RawMessage `json:"result"`
 	}
 	if err := json.Unmarshal(msg.Params, &subResult); err != nil {
-		log.Debug(fmt.Sprint("dropping invalid subscription message: ", msg))
+		log.Debug("dropping invalid subscription message", "msg", msg)
 		return
 	}
 	if c.subs[subResult.ID] != nil {
@@ -606,7 +647,7 @@ func (c *Client) handleNotification(msg *jsonrpcMessage) {
 func (c *Client) handleResponse(msg *jsonrpcMessage) {
 	op := c.respWait[string(msg.ID)]
 	if op == nil {
-		log.Debug(fmt.Sprintf("unsolicited response %v", msg))
+		log.Debug("unsolicited response", "msg", msg)
 		return
 	}
 	delete(c.respWait, string(msg.ID))

rpc/endpoints.go

@@ -0,0 +1,120 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"github.com/ethereum/go-ethereum/log"
+	"net"
+)
+
+// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules
+func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []string, vhosts []string) (net.Listener, *Server, error) {
+	// Generate the whitelist based on the allowed modules
+	whitelist := make(map[string]bool)
+	for _, module := range modules {
+		whitelist[module] = true
+	}
+	// Register all the APIs exposed by the services
+	handler := NewServer()
+	for _, api := range apis {
+		if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
+			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+				return nil, nil, err
+			}
+			log.Debug("HTTP registered", "namespace", api.Namespace)
+		}
+	}
+	// All APIs registered, start the HTTP listener
+	var (
+		listener net.Listener
+		err      error
+	)
+	if listener, err = net.Listen("tcp", endpoint); err != nil {
+		return nil, nil, err
+	}
+	go NewHTTPServer(cors, vhosts, handler).Serve(listener)
+	return listener, handler, err
+}
+
+// StartWSEndpoint starts a websocket endpoint
+func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *Server, error) {
+
+	// Generate the whitelist based on the allowed modules
+	whitelist := make(map[string]bool)
+	for _, module := range modules {
+		whitelist[module] = true
+	}
+	// Register all the APIs exposed by the services
+	handler := NewServer()
+	for _, api := range apis {
+		if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
+			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+				return nil, nil, err
+			}
+			log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
+		}
+	}
+	// All APIs registered, start the HTTP listener
+	var (
+		listener net.Listener
+		err      error
+	)
+	if listener, err = net.Listen("tcp", endpoint); err != nil {
+		return nil, nil, err
+	}
+	go NewWSServer(wsOrigins, handler).Serve(listener)
+	return listener, handler, err
+
+}
+
+// StartIPCEndpoint starts an IPC endpoint
+func StartIPCEndpoint(isClosedFn func() bool, ipcEndpoint string, apis []API) (net.Listener, *Server, error) {
+	// Register all the APIs exposed by the services
+	handler := NewServer()
+	for _, api := range apis {
+		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+			return nil, nil, err
+		}
+		log.Debug("IPC registered", "namespace", api.Namespace)
+	}
+	// All APIs registered, start the IPC listener
+	var (
+		listener net.Listener
+		err      error
+	)
+	if listener, err = CreateIPCListener(ipcEndpoint); err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				// Terminate if the listener was closed
+				if isClosedFn() {
+					log.Info("IPC closed", "err", err)
+				} else {
+					// Not closed, just some error; report and continue
+					log.Error("IPC accept failed", "err", err)
+				}
+				continue
+			}
+			go handler.ServeCodec(NewJSONCodec(conn), OptionMethodInvocation|OptionSubscriptions)
+		}
+	}()
+
+	return listener, handler, nil
+}

rpc/http.go

@@ -169,12 +169,17 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// All checks passed, create a codec that reads direct from the request body
 	// untilEOF and writes the response to w and order the server to process a
 	// single request.
+	ctx := context.Background()
+	ctx = context.WithValue(ctx, "remote", r.RemoteAddr)
+	ctx = context.WithValue(ctx, "scheme", r.Proto)
+	ctx = context.WithValue(ctx, "local", r.Host)
+
 	body := io.LimitReader(r.Body, maxRequestContentLength)
 	codec := NewJSONCodec(&httpReadWriteNopCloser{body, w})
 	defer codec.Close()
 
 	w.Header().Set("content-type", contentType)
-	srv.ServeSingleRequest(codec, OptionMethodInvocation)
+	srv.ServeSingleRequest(codec, OptionMethodInvocation, ctx)
 }
 
 // validateRequest returns a non-zero response code and error message if the

rpc/server.go

@@ -125,7 +125,7 @@ func (s *Server) RegisterName(name string, rcvr interface{}) error {
 // If singleShot is true it will process a single request, otherwise it will handle
 // requests until the codec returns an error when reading a request (in most cases
 // an EOF). It executes requests in parallel when singleShot is false.
-func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecOption) error {
+func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecOption, ctx context.Context) error {
 	var pend sync.WaitGroup
 
 	defer func() {
@@ -140,7 +140,8 @@ func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecO
 		s.codecsMu.Unlock()
 	}()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	//	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	// if the codec supports notification include a notifier that callbacks can use
@@ -215,14 +216,14 @@ func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecO
 // stopped. In either case the codec is closed.
 func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) {
 	defer codec.Close()
-	s.serveRequest(codec, false, options)
+	s.serveRequest(codec, false, options, context.Background())
 }
 
 // ServeSingleRequest reads and processes a single RPC request from the given codec. It will not
 // close the codec unless a non-recoverable error has occurred. Note, this method will return after
 // a single request has been processed!
-func (s *Server) ServeSingleRequest(codec ServerCodec, options CodecOption) {
-	s.serveRequest(codec, true, options)
+func (s *Server) ServeSingleRequest(codec ServerCodec, options CodecOption, ctx context.Context) {
+	s.serveRequest(codec, true, options, ctx)
 }
 
 // Stop will stop reading new requests, wait for stopPendingRequestTimeout to allow pending requests to finish,

signer/core/abihelper.go

@@ -0,0 +1,256 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+
+	"bytes"
+	"os"
+	"regexp"
+)
+
+type decodedArgument struct {
+	soltype abi.Argument
+	value   interface{}
+}
+type decodedCallData struct {
+	signature string
+	name      string
+	inputs    []decodedArgument
+}
+
+// String implements stringer interface, tries to use the underlying value-type
+func (arg decodedArgument) String() string {
+	var value string
+	switch arg.value.(type) {
+	case fmt.Stringer:
+		value = arg.value.(fmt.Stringer).String()
+	default:
+		value = fmt.Sprintf("%v", arg.value)
+	}
+	return fmt.Sprintf("%v: %v", arg.soltype.Type.String(), value)
+}
+
+// String implements stringer interface for decodedCallData
+func (cd decodedCallData) String() string {
+	args := make([]string, len(cd.inputs))
+	for i, arg := range cd.inputs {
+		args[i] = arg.String()
+	}
+	return fmt.Sprintf("%s(%s)", cd.name, strings.Join(args, ","))
+}
+
+// parseCallData matches the provided call data against the abi definition,
+// and returns a struct containing the actual go-typed values
+func parseCallData(calldata []byte, abidata string) (*decodedCallData, error) {
+
+	if len(calldata) < 4 {
+		return nil, fmt.Errorf("Invalid ABI-data, incomplete method signature of (%d bytes)", len(calldata))
+	}
+
+	sigdata, argdata := calldata[:4], calldata[4:]
+	if len(argdata)%32 != 0 {
+		return nil, fmt.Errorf("Not ABI-encoded data; length should be a multiple of 32 (was %d)", len(argdata))
+	}
+
+	abispec, err := abi.JSON(strings.NewReader(abidata))
+	if err != nil {
+		return nil, fmt.Errorf("Failed parsing JSON ABI: %v, abidata: %v", err, abidata)
+	}
+
+	method, err := abispec.MethodById(sigdata)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := method.Inputs.UnpackValues(argdata)
+	if err != nil {
+		return nil, err
+	}
+
+	decoded := decodedCallData{signature: method.Sig(), name: method.Name}
+
+	for n, argument := range method.Inputs {
+		if err != nil {
+			return nil, fmt.Errorf("Failed to decode argument %d (signature %v): %v", n, method.Sig(), err)
+		} else {
+			decodedArg := decodedArgument{
+				soltype: argument,
+				value:   v[n],
+			}
+			decoded.inputs = append(decoded.inputs, decodedArg)
+		}
+	}
+
+	// We're finished decoding the data. At this point, we encode the decoded data to see if it matches with the
+	// original data. If we didn't do that, it would e.g. be possible to stuff extra data into the arguments, which
+	// is not detected by merely decoding the data.
+
+	var (
+		encoded []byte
+	)
+	encoded, err = method.Inputs.PackValues(v)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !bytes.Equal(encoded, argdata) {
+		was := common.Bytes2Hex(encoded)
+		exp := common.Bytes2Hex(argdata)
+		return nil, fmt.Errorf("WARNING: Supplied data is stuffed with extra data. \nWant %s\nHave %s\nfor method %v", exp, was, method.Sig())
+	}
+	return &decoded, nil
+}
+
+// MethodSelectorToAbi converts a method selector into an ABI struct. The returned data is a valid json string
+// which can be consumed by the standard abi package.
+func MethodSelectorToAbi(selector string) ([]byte, error) {
+
+	re := regexp.MustCompile(`^([^\)]+)\(([a-z0-9,\[\]]*)\)`)
+
+	type fakeArg struct {
+		Type string `json:"type"`
+	}
+	type fakeABI struct {
+		Name   string    `json:"name"`
+		Type   string    `json:"type"`
+		Inputs []fakeArg `json:"inputs"`
+	}
+	groups := re.FindStringSubmatch(selector)
+	if len(groups) != 3 {
+		return nil, fmt.Errorf("Did not match: %v (%v matches)", selector, len(groups))
+	}
+	name := groups[1]
+	args := groups[2]
+	arguments := make([]fakeArg, 0)
+	if len(args) > 0 {
+		for _, arg := range strings.Split(args, ",") {
+			arguments = append(arguments, fakeArg{arg})
+		}
+	}
+	abicheat := fakeABI{
+		name, "function", arguments,
+	}
+	return json.Marshal([]fakeABI{abicheat})
+
+}
+
+type AbiDb struct {
+	db           map[string]string
+	customdb     map[string]string
+	customdbPath string
+}
+
+// NewEmptyAbiDB exists for test purposes
+func NewEmptyAbiDB() (*AbiDb, error) {
+	return &AbiDb{make(map[string]string), make(map[string]string), ""}, nil
+}
+
+// NewAbiDBFromFile loads signature database from file, and
+// errors if the file is not valid json. Does no other validation of contents
+func NewAbiDBFromFile(path string) (*AbiDb, error) {
+	raw, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	db, err := NewEmptyAbiDB()
+	if err != nil {
+		return nil, err
+	}
+	json.Unmarshal(raw, &db.db)
+	return db, nil
+}
+
+// NewAbiDBFromFiles loads both the standard signature database and a custom database. The latter will be used
+// to write new values into if they are submitted via the API
+func NewAbiDBFromFiles(standard, custom string) (*AbiDb, error) {
+
+	db := &AbiDb{make(map[string]string), make(map[string]string), custom}
+	db.customdbPath = custom
+
+	raw, err := ioutil.ReadFile(standard)
+	if err != nil {
+		return nil, err
+	}
+	json.Unmarshal(raw, &db.db)
+	// Custom file may not exist. Will be created during save, if needed
+	if _, err := os.Stat(custom); err == nil {
+		raw, err = ioutil.ReadFile(custom)
+		if err != nil {
+			return nil, err
+		}
+		json.Unmarshal(raw, &db.customdb)
+	}
+
+	return db, nil
+}
+
+// LookupMethodSelector checks the given 4byte-sequence against the known ABI methods.
+// OBS: This method does not validate the match, it's assumed the caller will do so
+func (db *AbiDb) LookupMethodSelector(id []byte) (string, error) {
+	if len(id) < 4 {
+		return "", fmt.Errorf("Expected 4-byte id, got %d", len(id))
+	}
+	sig := common.ToHex(id[:4])
+	if key, exists := db.db[sig]; exists {
+		return key, nil
+	}
+	if key, exists := db.customdb[sig]; exists {
+		return key, nil
+	}
+	return "", fmt.Errorf("Signature %v not found", sig)
+}
+func (db *AbiDb) Size() int {
+	return len(db.db)
+}
+
+// saveCustomAbi saves a signature ephemerally. If custom file is used, also saves to disk
+func (db *AbiDb) saveCustomAbi(selector, signature string) error {
+	db.customdb[signature] = selector
+	if db.customdbPath == "" {
+		return nil //Not an error per se, just not used
+	}
+	d, err := json.Marshal(db.customdb)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(db.customdbPath, d, 0600)
+	return err
+}
+
+// Adds a signature to the database, if custom database saving is enabled.
+// OBS: This method does _not_ validate the correctness of the data,
+// it is assumed that the caller has already done so
+func (db *AbiDb) AddSignature(selector string, data []byte) error {
+	if len(data) < 4 {
+		return nil
+	}
+	_, err := db.LookupMethodSelector(data[:4])
+	if err == nil {
+		return nil
+	}
+	sig := common.ToHex(data[:4])
+	return db.saveCustomAbi(selector, sig)
+}

signer/core/abihelper_test.go

@@ -0,0 +1,247 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"io/ioutil"
+	"math/big"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func verify(t *testing.T, jsondata, calldata string, exp []interface{}) {
+
+	abispec, err := abi.JSON(strings.NewReader(jsondata))
+	if err != nil {
+		t.Fatal(err)
+	}
+	cd := common.Hex2Bytes(calldata)
+	sigdata, argdata := cd[:4], cd[4:]
+	method, err := abispec.MethodById(sigdata)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data, err := method.Inputs.UnpackValues(argdata)
+
+	if len(data) != len(exp) {
+		t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data))
+	}
+	for i, elem := range data {
+		if !reflect.DeepEqual(elem, exp[i]) {
+			t.Fatalf("Unpack error, arg %d, got %v, want %v", i, elem, exp[i])
+		}
+	}
+}
+func TestNewUnpacker(t *testing.T) {
+	type unpackTest struct {
+		jsondata string
+		calldata string
+		exp      []interface{}
+	}
+	testcases := []unpackTest{
+		{ // https://solidity.readthedocs.io/en/develop/abi-spec.html#use-of-dynamic-types
+			`[{"type":"function","name":"f", "inputs":[{"type":"uint256"},{"type":"uint32[]"},{"type":"bytes10"},{"type":"bytes"}]}]`,
+			// 0x123, [0x456, 0x789], "1234567890", "Hello, world!"
+			"8be65246" + "00000000000000000000000000000000000000000000000000000000000001230000000000000000000000000000000000000000000000000000000000000080313233343536373839300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000004560000000000000000000000000000000000000000000000000000000000000789000000000000000000000000000000000000000000000000000000000000000d48656c6c6f2c20776f726c642100000000000000000000000000000000000000",
+			[]interface{}{
+				big.NewInt(0x123),
+				[]uint32{0x456, 0x789},
+				[10]byte{49, 50, 51, 52, 53, 54, 55, 56, 57, 48},
+				common.Hex2Bytes("48656c6c6f2c20776f726c6421"),
+			},
+		}, { // https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI#examples
+			`[{"type":"function","name":"sam","inputs":[{"type":"bytes"},{"type":"bool"},{"type":"uint256[]"}]}]`,
+			//  "dave", true and [1,2,3]
+			"a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+			[]interface{}{
+				[]byte{0x64, 0x61, 0x76, 0x65},
+				true,
+				[]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
+			},
+		}, {
+			`[{"type":"function","name":"send","inputs":[{"type":"uint256"}]}]`,
+			"a52c101e0000000000000000000000000000000000000000000000000000000000000012",
+			[]interface{}{big.NewInt(0x12)},
+		}, {
+			`[{"type":"function","name":"compareAndApprove","inputs":[{"type":"address"},{"type":"uint256"},{"type":"uint256"}]}]`,
+			"751e107900000000000000000000000000000133700000deadbeef00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
+			[]interface{}{
+				common.HexToAddress("0x00000133700000deadbeef000000000000000000"),
+				new(big.Int).SetBytes([]byte{0x00}),
+				big.NewInt(0x1),
+			},
+		},
+	}
+	for _, c := range testcases {
+		verify(t, c.jsondata, c.calldata, c.exp)
+	}
+
+}
+
+/*
+func TestReflect(t *testing.T) {
+	a := big.NewInt(0)
+	b := new(big.Int).SetBytes([]byte{0x00})
+	if !reflect.DeepEqual(a, b) {
+		t.Fatalf("Nope, %v != %v", a, b)
+	}
+}
+*/
+
+func TestCalldataDecoding(t *testing.T) {
+
+	// send(uint256)                              : a52c101e
+	// compareAndApprove(address,uint256,uint256) : 751e1079
+	// issue(address[],uint256)                   : 42958b54
+	jsondata := `
+[
+	{"type":"function","name":"send","inputs":[{"name":"a","type":"uint256"}]},
+	{"type":"function","name":"compareAndApprove","inputs":[{"name":"a","type":"address"},{"name":"a","type":"uint256"},{"name":"a","type":"uint256"}]},
+	{"type":"function","name":"issue","inputs":[{"name":"a","type":"address[]"},{"name":"a","type":"uint256"}]},
+	{"type":"function","name":"sam","inputs":[{"name":"a","type":"bytes"},{"name":"a","type":"bool"},{"name":"a","type":"uint256[]"}]}
+]`
+	//Expected failures
+	for _, hexdata := range []string{
+		"a52c101e00000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000042",
+		"a52c101e000000000000000000000000000000000000000000000000000000000000001200",
+		"a52c101e00000000000000000000000000000000000000000000000000000000000000",
+		"a52c101e",
+		"a52c10",
+		"",
+		// Too short
+		"751e10790000000000000000000000000000000000000000000000000000000000000012",
+		"751e1079FFffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		//Not valid multiple of 32
+		"deadbeef00000000000000000000000000000000000000000000000000000000000000",
+		//Too short 'issue'
+		"42958b5400000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000042",
+		// Too short compareAndApprove
+		"a52c101e00ff0000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000042",
+		// From https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI
+		// contains a bool with illegal values
+		"a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+	} {
+		_, err := parseCallData(common.Hex2Bytes(hexdata), jsondata)
+		if err == nil {
+			t.Errorf("Expected decoding to fail: %s", hexdata)
+		}
+	}
+
+	//Expected success
+	for _, hexdata := range []string{
+		// From https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI
+		"a5643bf20000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000464617665000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		"a52c101e0000000000000000000000000000000000000000000000000000000000000012",
+		"a52c101eFFffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		"751e1079000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+		"42958b54" +
+			// start of dynamic type
+			"0000000000000000000000000000000000000000000000000000000000000040" +
+			//uint256
+			"0000000000000000000000000000000000000000000000000000000000000001" +
+			// length of  array
+			"0000000000000000000000000000000000000000000000000000000000000002" +
+			// array values
+			"000000000000000000000000000000000000000000000000000000000000dead" +
+			"000000000000000000000000000000000000000000000000000000000000beef",
+	} {
+		_, err := parseCallData(common.Hex2Bytes(hexdata), jsondata)
+		if err != nil {
+			t.Errorf("Unexpected failure on input %s:\n %v (%d bytes) ", hexdata, err, len(common.Hex2Bytes(hexdata)))
+		}
+	}
+}
+
+func TestSelectorUnmarshalling(t *testing.T) {
+	var (
+		db        *AbiDb
+		err       error
+		abistring []byte
+		abistruct abi.ABI
+	)
+
+	db, err = NewAbiDBFromFile("../../cmd/clef/4byte.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Printf("DB size %v\n", db.Size())
+	for id, selector := range db.db {
+
+		abistring, err = MethodSelectorToAbi(selector)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		abistruct, err = abi.JSON(strings.NewReader(string(abistring)))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		m, err := abistruct.MethodById(common.Hex2Bytes(id[2:]))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if m.Sig() != selector {
+			t.Errorf("Expected equality: %v != %v", m.Sig(), selector)
+		}
+	}
+
+}
+
+func TestCustomABI(t *testing.T) {
+	d, err := ioutil.TempDir("", "signer-4byte-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	filename := fmt.Sprintf("%s/4byte_custom.json", d)
+	abidb, err := NewAbiDBFromFiles("../../cmd/clef/4byte.json", filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Now we'll remove all existing signatures
+	abidb.db = make(map[string]string)
+	calldata := common.Hex2Bytes("a52c101edeadbeef")
+	_, err = abidb.LookupMethodSelector(calldata)
+	if err == nil {
+		t.Fatalf("Should not find a match on empty db")
+	}
+	if err = abidb.AddSignature("send(uint256)", calldata); err != nil {
+		t.Fatalf("Failed to save file: %v", err)
+	}
+	_, err = abidb.LookupMethodSelector(calldata)
+	if err != nil {
+		t.Fatalf("Should find a match for abi signature, got: %v", err)
+	}
+	//Check that it wrote to file
+	abidb2, err := NewAbiDBFromFile(filename)
+	if err != nil {
+		t.Fatalf("Failed to create new abidb: %v", err)
+	}
+	_, err = abidb2.LookupMethodSelector(calldata)
+	if err != nil {
+		t.Fatalf("Save failed: should find a match for abi signature after loading from disk")
+	}
+}

signer/core/api.go

@@ -0,0 +1,500 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/accounts/usbwallet"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// ExternalAPI defines the external API through which signing requests are made.
+type ExternalAPI interface {
+	// List available accounts
+	List(ctx context.Context) (Accounts, error)
+	// New request to create a new account
+	New(ctx context.Context) (accounts.Account, error)
+	// SignTransaction request to sign the specified transaction
+	SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error)
+	// Sign - request to sign the given data (plus prefix)
+	Sign(ctx context.Context, addr common.MixedcaseAddress, data hexutil.Bytes) (hexutil.Bytes, error)
+	// EcRecover - request to perform ecrecover
+	EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error)
+	// Export - request to export an account
+	Export(ctx context.Context, addr common.Address) (json.RawMessage, error)
+	// Import - request to import an account
+	Import(ctx context.Context, keyJSON json.RawMessage) (Account, error)
+}
+
+// SignerUI specifies what method a UI needs to implement to be able to be used as a UI for the signer
+type SignerUI interface {
+	// ApproveTx prompt the user for confirmation to request to sign Transaction
+	ApproveTx(request *SignTxRequest) (SignTxResponse, error)
+	// ApproveSignData prompt the user for confirmation to request to sign data
+	ApproveSignData(request *SignDataRequest) (SignDataResponse, error)
+	// ApproveExport prompt the user for confirmation to export encrypted Account json
+	ApproveExport(request *ExportRequest) (ExportResponse, error)
+	// ApproveImport prompt the user for confirmation to import Account json
+	ApproveImport(request *ImportRequest) (ImportResponse, error)
+	// ApproveListing prompt the user for confirmation to list accounts
+	// the list of accounts to list can be modified by the UI
+	ApproveListing(request *ListRequest) (ListResponse, error)
+	// ApproveNewAccount prompt the user for confirmation to create new Account, and reveal to caller
+	ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error)
+	// ShowError displays error message to user
+	ShowError(message string)
+	// ShowInfo displays info message to user
+	ShowInfo(message string)
+	// OnApprovedTx notifies the UI about a transaction having been successfully signed.
+	// This method can be used by a UI to keep track of e.g. how much has been sent to a particular recipient.
+	OnApprovedTx(tx ethapi.SignTransactionResult)
+	// OnSignerStartup is invoked when the signer boots, and tells the UI info about external API location and version
+	// information
+	OnSignerStartup(info StartupInfo)
+}
+
+// SignerAPI defines the actual implementation of ExternalAPI
+type SignerAPI struct {
+	chainID   *big.Int
+	am        *accounts.Manager
+	UI        SignerUI
+	validator *Validator
+}
+
+// Metadata about a request
+type Metadata struct {
+	Remote string `json:"remote"`
+	Local  string `json:"local"`
+	Scheme string `json:"scheme"`
+}
+
+// MetadataFromContext extracts Metadata from a given context.Context
+func MetadataFromContext(ctx context.Context) Metadata {
+	m := Metadata{"NA", "NA", "NA"} // batman
+
+	if v := ctx.Value("remote"); v != nil {
+		m.Remote = v.(string)
+	}
+	if v := ctx.Value("scheme"); v != nil {
+		m.Scheme = v.(string)
+	}
+	if v := ctx.Value("local"); v != nil {
+		m.Local = v.(string)
+	}
+	return m
+}
+
+// String implements Stringer interface
+func (m Metadata) String() string {
+	s, err := json.Marshal(m)
+	if err == nil {
+		return string(s)
+	}
+	return err.Error()
+}
+
+// types for the requests/response types between signer and UI
+type (
+	// SignTxRequest contains info about a Transaction to sign
+	SignTxRequest struct {
+		Transaction SendTxArgs       `json:"transaction"`
+		Callinfo    []ValidationInfo `json:"call_info"`
+		Meta        Metadata         `json:"meta"`
+	}
+	// SignTxResponse result from SignTxRequest
+	SignTxResponse struct {
+		//The UI may make changes to the TX
+		Transaction SendTxArgs `json:"transaction"`
+		Approved    bool       `json:"approved"`
+		Password    string     `json:"password"`
+	}
+	// ExportRequest info about query to export accounts
+	ExportRequest struct {
+		Address common.Address `json:"address"`
+		Meta    Metadata       `json:"meta"`
+	}
+	// ExportResponse response to export-request
+	ExportResponse struct {
+		Approved bool `json:"approved"`
+	}
+	// ImportRequest info about request to import an Account
+	ImportRequest struct {
+		Meta Metadata `json:"meta"`
+	}
+	ImportResponse struct {
+		Approved    bool   `json:"approved"`
+		OldPassword string `json:"old_password"`
+		NewPassword string `json:"new_password"`
+	}
+	SignDataRequest struct {
+		Address common.MixedcaseAddress `json:"address"`
+		Rawdata hexutil.Bytes           `json:"raw_data"`
+		Message string                  `json:"message"`
+		Hash    hexutil.Bytes           `json:"hash"`
+		Meta    Metadata                `json:"meta"`
+	}
+	SignDataResponse struct {
+		Approved bool `json:"approved"`
+		Password string
+	}
+	NewAccountRequest struct {
+		Meta Metadata `json:"meta"`
+	}
+	NewAccountResponse struct {
+		Approved bool   `json:"approved"`
+		Password string `json:"password"`
+	}
+	ListRequest struct {
+		Accounts []Account `json:"accounts"`
+		Meta     Metadata  `json:"meta"`
+	}
+	ListResponse struct {
+		Accounts []Account `json:"accounts"`
+	}
+	Message struct {
+		Text string `json:"text"`
+	}
+	StartupInfo struct {
+		Info map[string]interface{} `json:"info"`
+	}
+)
+
+var ErrRequestDenied = errors.New("Request denied")
+
+type errorWrapper struct {
+	msg string
+	err error
+}
+
+func (ew errorWrapper) String() string {
+	return fmt.Sprintf("%s\n%s", ew.msg, ew.err)
+}
+
+// NewSignerAPI creates a new API that can be used for Account management.
+// ksLocation specifies the directory where to store the password protected private
+// key that is generated when a new Account is created.
+// noUSB disables USB support that is required to support hardware devices such as
+// ledger and trezor.
+func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abidb *AbiDb, lightKDF bool) *SignerAPI {
+	var (
+		backends []accounts.Backend
+		n, p     = keystore.StandardScryptN, keystore.StandardScryptP
+	)
+	if lightKDF {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	// support password based accounts
+	if len(ksLocation) > 0 {
+		backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
+	}
+	if !noUSB {
+		// Start a USB hub for Ledger hardware wallets
+		if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
+		} else {
+			backends = append(backends, ledgerhub)
+			log.Debug("Ledger support enabled")
+		}
+		// Start a USB hub for Trezor hardware wallets
+		if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
+		} else {
+			backends = append(backends, trezorhub)
+			log.Debug("Trezor support enabled")
+		}
+	}
+	return &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb)}
+}
+
+// List returns the set of wallet this signer manages. Each wallet can contain
+// multiple accounts.
+func (api *SignerAPI) List(ctx context.Context) (Accounts, error) {
+	var accs []Account
+	for _, wallet := range api.am.Wallets() {
+		for _, acc := range wallet.Accounts() {
+			acc := Account{Typ: "Account", URL: wallet.URL(), Address: acc.Address}
+			accs = append(accs, acc)
+		}
+	}
+	result, err := api.UI.ApproveListing(&ListRequest{Accounts: accs, Meta: MetadataFromContext(ctx)})
+	if err != nil {
+		return nil, err
+	}
+	if result.Accounts == nil {
+		return nil, ErrRequestDenied
+
+	}
+	return result.Accounts, nil
+}
+
+// New creates a new password protected Account. The private key is protected with
+// the given password. Users are responsible to backup the private key that is stored
+// in the keystore location thas was specified when this API was created.
+func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
+	be := api.am.Backends(keystore.KeyStoreType)
+	if len(be) == 0 {
+		return accounts.Account{}, errors.New("password based accounts not supported")
+	}
+	resp, err := api.UI.ApproveNewAccount(&NewAccountRequest{MetadataFromContext(ctx)})
+
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if !resp.Approved {
+		return accounts.Account{}, ErrRequestDenied
+	}
+	return be[0].(*keystore.KeyStore).NewAccount(resp.Password)
+}
+
+// logDiff logs the difference between the incoming (original) transaction and the one returned from the signer.
+// it also returns 'true' if the transaction was modified, to make it possible to configure the signer not to allow
+// UI-modifications to requests
+func logDiff(original *SignTxRequest, new *SignTxResponse) bool {
+	modified := false
+	if f0, f1 := original.Transaction.From, new.Transaction.From; !reflect.DeepEqual(f0, f1) {
+		log.Info("Sender-account changed by UI", "was", f0, "is", f1)
+		modified = true
+	}
+	if t0, t1 := original.Transaction.To, new.Transaction.To; !reflect.DeepEqual(t0, t1) {
+		log.Info("Recipient-account changed by UI", "was", t0, "is", t1)
+		modified = true
+	}
+	if g0, g1 := original.Transaction.Gas, new.Transaction.Gas; g0 != g1 {
+		modified = true
+		log.Info("Gas changed by UI", "was", g0, "is", g1)
+	}
+	if g0, g1 := big.Int(original.Transaction.GasPrice), big.Int(new.Transaction.GasPrice); g0.Cmp(&g1) != 0 {
+		modified = true
+		log.Info("GasPrice changed by UI", "was", g0, "is", g1)
+	}
+	if v0, v1 := big.Int(original.Transaction.Value), big.Int(new.Transaction.Value); v0.Cmp(&v1) != 0 {
+		modified = true
+		log.Info("Value changed by UI", "was", v0, "is", v1)
+	}
+	if d0, d1 := original.Transaction.Data, new.Transaction.Data; d0 != d1 {
+		d0s := ""
+		d1s := ""
+		if d0 != nil {
+			d0s = common.ToHex(*d0)
+		}
+		if d1 != nil {
+			d1s = common.ToHex(*d1)
+		}
+		if d1s != d0s {
+			modified = true
+			log.Info("Data changed by UI", "was", d0s, "is", d1s)
+		}
+	}
+	if n0, n1 := original.Transaction.Nonce, new.Transaction.Nonce; n0 != n1 {
+		modified = true
+		log.Info("Nonce changed by UI", "was", n0, "is", n1)
+	}
+	return modified
+}
+
+// SignTransaction signs the given Transaction and returns it both as json and rlp-encoded form
+func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) {
+	var (
+		err    error
+		result SignTxResponse
+	)
+	msgs, err := api.validator.ValidateTransaction(&args, methodSelector)
+	if err != nil {
+		return nil, err
+	}
+
+	req := SignTxRequest{
+		Transaction: args,
+		Meta:        MetadataFromContext(ctx),
+		Callinfo:    msgs.Messages,
+	}
+	// Process approval
+	result, err = api.UI.ApproveTx(&req)
+	if err != nil {
+		return nil, err
+	}
+	if !result.Approved {
+		return nil, ErrRequestDenied
+	}
+	// Log changes made by the UI to the signing-request
+	logDiff(&req, &result)
+	var (
+		acc    accounts.Account
+		wallet accounts.Wallet
+	)
+	acc = accounts.Account{Address: result.Transaction.From.Address()}
+	wallet, err = api.am.Find(acc)
+	if err != nil {
+		return nil, err
+	}
+	// Convert fields into a real transaction
+	var unsignedTx = result.Transaction.toTransaction()
+
+	// The one to sign is the one that was returned from the UI
+	signedTx, err := wallet.SignTxWithPassphrase(acc, result.Password, unsignedTx, api.chainID)
+	if err != nil {
+		api.UI.ShowError(err.Error())
+		return nil, err
+	}
+
+	rlpdata, err := rlp.EncodeToBytes(signedTx)
+	response := ethapi.SignTransactionResult{Raw: rlpdata, Tx: signedTx}
+
+	// Finally, send the signed tx to the UI
+	api.UI.OnApprovedTx(response)
+	// ...and to the external caller
+	return &response, nil
+
+}
+
+// Sign calculates an Ethereum ECDSA signature for:
+// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message))
+//
+// Note, the produced signature conforms to the secp256k1 curve R, S and V values,
+// where the V value will be 27 or 28 for legacy reasons.
+//
+// The key used to calculate the signature is decrypted with the given password.
+//
+// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign
+func (api *SignerAPI) Sign(ctx context.Context, addr common.MixedcaseAddress, data hexutil.Bytes) (hexutil.Bytes, error) {
+	sighash, msg := SignHash(data)
+	// We make the request prior to looking up if we actually have the account, to prevent
+	// account-enumeration via the API
+	req := &SignDataRequest{Address: addr, Rawdata: data, Message: msg, Hash: sighash, Meta: MetadataFromContext(ctx)}
+	res, err := api.UI.ApproveSignData(req)
+
+	if err != nil {
+		return nil, err
+	}
+	if !res.Approved {
+		return nil, ErrRequestDenied
+	}
+	// Look up the wallet containing the requested signer
+	account := accounts.Account{Address: addr.Address()}
+	wallet, err := api.am.Find(account)
+	if err != nil {
+		return nil, err
+	}
+	// Assemble sign the data with the wallet
+	signature, err := wallet.SignHashWithPassphrase(account, res.Password, sighash)
+	if err != nil {
+		api.UI.ShowError(err.Error())
+		return nil, err
+	}
+	signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
+	return signature, nil
+}
+
+// EcRecover returns the address for the Account that was used to create the signature.
+// Note, this function is compatible with eth_sign and personal_sign. As such it recovers
+// the address of:
+// hash = keccak256("\x19Ethereum Signed Message:\n"${message length}${message})
+// addr = ecrecover(hash, signature)
+//
+// Note, the signature must conform to the secp256k1 curve R, S and V values, where
+// the V value must be be 27 or 28 for legacy reasons.
+//
+// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover
+func (api *SignerAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) {
+	if len(sig) != 65 {
+		return common.Address{}, fmt.Errorf("signature must be 65 bytes long")
+	}
+	if sig[64] != 27 && sig[64] != 28 {
+		return common.Address{}, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)")
+	}
+	sig[64] -= 27 // Transform yellow paper V from 27/28 to 0/1
+	hash, _ := SignHash(data)
+	rpk, err := crypto.Ecrecover(hash, sig)
+	if err != nil {
+		return common.Address{}, err
+	}
+	pubKey := crypto.ToECDSAPub(rpk)
+	recoveredAddr := crypto.PubkeyToAddress(*pubKey)
+	return recoveredAddr, nil
+}
+
+// SignHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calculated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func SignHash(data []byte) ([]byte, string) {
+	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
+	return crypto.Keccak256([]byte(msg)), msg
+}
+
+// Export returns encrypted private key associated with the given address in web3 keystore format.
+func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
+	res, err := api.UI.ApproveExport(&ExportRequest{Address: addr, Meta: MetadataFromContext(ctx)})
+
+	if err != nil {
+		return nil, err
+	}
+	if !res.Approved {
+		return nil, ErrRequestDenied
+	}
+	// Look up the wallet containing the requested signer
+	wallet, err := api.am.Find(accounts.Account{Address: addr})
+	if err != nil {
+		return nil, err
+	}
+	if wallet.URL().Scheme != keystore.KeyStoreScheme {
+		return nil, fmt.Errorf("Account is not a keystore-account")
+	}
+	return ioutil.ReadFile(wallet.URL().Path)
+}
+
+// Imports tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
+// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
+// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
+func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) {
+	be := api.am.Backends(keystore.KeyStoreType)
+
+	if len(be) == 0 {
+		return Account{}, errors.New("password based accounts not supported")
+	}
+	res, err := api.UI.ApproveImport(&ImportRequest{Meta: MetadataFromContext(ctx)})
+
+	if err != nil {
+		return Account{}, err
+	}
+	if !res.Approved {
+		return Account{}, ErrRequestDenied
+	}
+	acc, err := be[0].(*keystore.KeyStore).Import(keyJSON, res.OldPassword, res.NewPassword)
+	if err != nil {
+		api.UI.ShowError(err.Error())
+		return Account{}, err
+	}
+	return Account{Typ: "Account", URL: acc.URL, Address: acc.Address}, nil
+}

signer/core/api_test.go

@@ -0,0 +1,386 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package core
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+//Used for testing
+type HeadlessUI struct {
+	controller chan string
+}
+
+func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) {
+}
+
+func (ui *HeadlessUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	fmt.Printf("OnApproved called")
+}
+
+func (ui *HeadlessUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
+
+	switch <-ui.controller {
+	case "Y":
+		return SignTxResponse{request.Transaction, true, <-ui.controller}, nil
+	case "M": //Modify
+		old := big.Int(request.Transaction.Value)
+		newVal := big.NewInt(0).Add(&old, big.NewInt(1))
+		request.Transaction.Value = hexutil.Big(*newVal)
+		return SignTxResponse{request.Transaction, true, <-ui.controller}, nil
+	default:
+		return SignTxResponse{request.Transaction, false, ""}, nil
+	}
+}
+func (ui *HeadlessUI) ApproveSignData(request *SignDataRequest) (SignDataResponse, error) {
+	if "Y" == <-ui.controller {
+		return SignDataResponse{true, <-ui.controller}, nil
+	}
+	return SignDataResponse{false, ""}, nil
+}
+func (ui *HeadlessUI) ApproveExport(request *ExportRequest) (ExportResponse, error) {
+
+	return ExportResponse{<-ui.controller == "Y"}, nil
+
+}
+func (ui *HeadlessUI) ApproveImport(request *ImportRequest) (ImportResponse, error) {
+
+	if "Y" == <-ui.controller {
+		return ImportResponse{true, <-ui.controller, <-ui.controller}, nil
+	}
+	return ImportResponse{false, "", ""}, nil
+}
+func (ui *HeadlessUI) ApproveListing(request *ListRequest) (ListResponse, error) {
+
+	switch <-ui.controller {
+	case "A":
+		return ListResponse{request.Accounts}, nil
+	case "1":
+		l := make([]Account, 1)
+		l[0] = request.Accounts[1]
+		return ListResponse{l}, nil
+	default:
+		return ListResponse{nil}, nil
+	}
+}
+func (ui *HeadlessUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) {
+
+	if "Y" == <-ui.controller {
+		return NewAccountResponse{true, <-ui.controller}, nil
+	}
+	return NewAccountResponse{false, ""}, nil
+}
+func (ui *HeadlessUI) ShowError(message string) {
+	//stdout is used by communication
+	fmt.Fprint(os.Stderr, message)
+}
+func (ui *HeadlessUI) ShowInfo(message string) {
+	//stdout is used by communication
+	fmt.Fprint(os.Stderr, message)
+}
+
+func tmpDirName(t *testing.T) string {
+	d, err := ioutil.TempDir("", "eth-keystore-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err = filepath.EvalSymlinks(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+func setup(t *testing.T) (*SignerAPI, chan string) {
+
+	controller := make(chan string, 10)
+
+	db, err := NewAbiDBFromFile("../../cmd/clef/4byte.json")
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	var (
+		ui  = &HeadlessUI{controller}
+		api = NewSignerAPI(
+			1,
+			tmpDirName(t),
+			true,
+			ui,
+			db,
+			true)
+	)
+	return api, controller
+}
+func createAccount(control chan string, api *SignerAPI, t *testing.T) {
+
+	control <- "Y"
+	control <- "apassword"
+	_, err := api.New(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Some time to allow changes to propagate
+	time.Sleep(250 * time.Millisecond)
+}
+func failCreateAccount(control chan string, api *SignerAPI, t *testing.T) {
+	control <- "N"
+	acc, err := api.New(context.Background())
+	if err != ErrRequestDenied {
+		t.Fatal(err)
+	}
+	if acc.Address != (common.Address{}) {
+		t.Fatal("Empty address should be returned")
+	}
+}
+func list(control chan string, api *SignerAPI, t *testing.T) []Account {
+	control <- "A"
+	list, err := api.List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return list
+}
+
+func TestNewAcc(t *testing.T) {
+
+	api, control := setup(t)
+	verifyNum := func(num int) {
+		if list := list(control, api, t); len(list) != num {
+			t.Errorf("Expected %d accounts, got %d", num, len(list))
+		}
+	}
+	// Testing create and create-deny
+	createAccount(control, api, t)
+	createAccount(control, api, t)
+	failCreateAccount(control, api, t)
+	failCreateAccount(control, api, t)
+	createAccount(control, api, t)
+	failCreateAccount(control, api, t)
+	createAccount(control, api, t)
+	failCreateAccount(control, api, t)
+	verifyNum(4)
+
+	// Testing listing:
+	// Listing one Account
+	control <- "1"
+	list, err := api.List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(list) != 1 {
+		t.Fatalf("List should only show one Account")
+	}
+	// Listing denied
+	control <- "Nope"
+	list, err = api.List(context.Background())
+	if len(list) != 0 {
+		t.Fatalf("List should be empty")
+	}
+	if err != ErrRequestDenied {
+		t.Fatal("Expected deny")
+	}
+}
+
+func TestSignData(t *testing.T) {
+
+	api, control := setup(t)
+	//Create two accounts
+	createAccount(control, api, t)
+	createAccount(control, api, t)
+	control <- "1"
+	list, err := api.List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := common.NewMixedcaseAddress(list[0].Address)
+
+	control <- "Y"
+	control <- "wrongpassword"
+	h, err := api.Sign(context.Background(), a, []byte("EHLO world"))
+	if h != nil {
+		t.Errorf("Expected nil-data, got %x", h)
+	}
+	if err != keystore.ErrDecrypt {
+		t.Errorf("Expected ErrLocked! %v", err)
+	}
+
+	control <- "No way"
+	h, err = api.Sign(context.Background(), a, []byte("EHLO world"))
+	if h != nil {
+		t.Errorf("Expected nil-data, got %x", h)
+	}
+	if err != ErrRequestDenied {
+		t.Errorf("Expected ErrRequestDenied! %v", err)
+	}
+
+	control <- "Y"
+	control <- "apassword"
+	h, err = api.Sign(context.Background(), a, []byte("EHLO world"))
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	if h == nil || len(h) != 65 {
+		t.Errorf("Expected 65 byte signature (got %d bytes)", len(h))
+	}
+}
+func mkTestTx(from common.MixedcaseAddress) SendTxArgs {
+	to := common.NewMixedcaseAddress(common.HexToAddress("0x1337"))
+	gas := hexutil.Uint64(21000)
+	gasPrice := (hexutil.Big)(*big.NewInt(2000000000))
+	value := (hexutil.Big)(*big.NewInt(1e18))
+	nonce := (hexutil.Uint64)(0)
+	data := hexutil.Bytes(common.Hex2Bytes("01020304050607080a"))
+	tx := SendTxArgs{
+		From:     from,
+		To:       &to,
+		Gas:      gas,
+		GasPrice: gasPrice,
+		Value:    value,
+		Data:     &data,
+		Nonce:    nonce}
+	return tx
+}
+
+func TestSignTx(t *testing.T) {
+
+	var (
+		list      Accounts
+		res, res2 *ethapi.SignTransactionResult
+		err       error
+	)
+
+	api, control := setup(t)
+	createAccount(control, api, t)
+	control <- "A"
+	list, err = api.List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := common.NewMixedcaseAddress(list[0].Address)
+
+	methodSig := "test(uint)"
+	tx := mkTestTx(a)
+
+	control <- "Y"
+	control <- "wrongpassword"
+	res, err = api.SignTransaction(context.Background(), tx, &methodSig)
+	if res != nil {
+		t.Errorf("Expected nil-response, got %v", res)
+	}
+	if err != keystore.ErrDecrypt {
+		t.Errorf("Expected ErrLocked! %v", err)
+	}
+
+	control <- "No way"
+	res, err = api.SignTransaction(context.Background(), tx, &methodSig)
+	if res != nil {
+		t.Errorf("Expected nil-response, got %v", res)
+	}
+	if err != ErrRequestDenied {
+		t.Errorf("Expected ErrRequestDenied! %v", err)
+	}
+
+	control <- "Y"
+	control <- "apassword"
+	res, err = api.SignTransaction(context.Background(), tx, &methodSig)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	parsedTx := &types.Transaction{}
+	rlp.Decode(bytes.NewReader(res.Raw), parsedTx)
+	//The tx should NOT be modified by the UI
+	if parsedTx.Value().Cmp(tx.Value.ToInt()) != 0 {
+		t.Errorf("Expected value to be unchanged, expected %v got %v", tx.Value, parsedTx.Value())
+	}
+	control <- "Y"
+	control <- "apassword"
+
+	res2, err = api.SignTransaction(context.Background(), tx, &methodSig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(res.Raw, res2.Raw) {
+		t.Error("Expected tx to be unmodified by UI")
+	}
+
+	//The tx is modified by the UI
+	control <- "M"
+	control <- "apassword"
+
+	res2, err = api.SignTransaction(context.Background(), tx, &methodSig)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parsedTx2 := &types.Transaction{}
+	rlp.Decode(bytes.NewReader(res.Raw), parsedTx2)
+	//The tx should be modified by the UI
+	if parsedTx2.Value().Cmp(tx.Value.ToInt()) != 0 {
+		t.Errorf("Expected value to be unchanged, got %v", parsedTx.Value())
+	}
+
+	if bytes.Equal(res.Raw, res2.Raw) {
+		t.Error("Expected tx to be modified by UI")
+	}
+
+}
+
+/*
+func TestAsyncronousResponses(t *testing.T){
+
+	//Set up one account
+	api, control := setup(t)
+	createAccount(control, api, t)
+
+	// Two transactions, the second one with larger value than the first
+	tx1 := mkTestTx()
+	newVal := big.NewInt(0).Add((*big.Int) (tx1.Value), big.NewInt(1))
+	tx2 := mkTestTx()
+	tx2.Value = (*hexutil.Big)(newVal)
+
+	control <- "W" //wait
+	control <- "Y" //
+	control <- "apassword"
+	control <- "Y" //
+	control <- "apassword"
+
+	var err error
+
+	h1, err := api.SignTransaction(context.Background(), common.HexToAddress("1111"), tx1, nil)
+	h2, err := api.SignTransaction(context.Background(), common.HexToAddress("2222"), tx2, nil)
+
+
+	}
+*/

signer/core/auditlog.go

@@ -0,0 +1,110 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"context"
+
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type AuditLogger struct {
+	log log.Logger
+	api ExternalAPI
+}
+
+func (l *AuditLogger) List(ctx context.Context) (Accounts, error) {
+	l.log.Info("List", "type", "request", "metadata", MetadataFromContext(ctx).String())
+	res, e := l.api.List(ctx)
+
+	l.log.Info("List", "type", "response", "data", res.String())
+
+	return res, e
+}
+
+func (l *AuditLogger) New(ctx context.Context) (accounts.Account, error) {
+	return l.api.New(ctx)
+}
+
+func (l *AuditLogger) SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) {
+	sel := "<nil>"
+	if methodSelector != nil {
+		sel = *methodSelector
+	}
+	l.log.Info("SignTransaction", "type", "request", "metadata", MetadataFromContext(ctx).String(),
+		"tx", args.String(),
+		"methodSelector", sel)
+
+	res, e := l.api.SignTransaction(ctx, args, methodSelector)
+	if res != nil {
+		l.log.Info("SignTransaction", "type", "response", "data", common.Bytes2Hex(res.Raw), "error", e)
+	} else {
+		l.log.Info("SignTransaction", "type", "response", "data", res, "error", e)
+	}
+	return res, e
+}
+
+func (l *AuditLogger) Sign(ctx context.Context, addr common.MixedcaseAddress, data hexutil.Bytes) (hexutil.Bytes, error) {
+	l.log.Info("Sign", "type", "request", "metadata", MetadataFromContext(ctx).String(),
+		"addr", addr.String(), "data", common.Bytes2Hex(data))
+	b, e := l.api.Sign(ctx, addr, data)
+	l.log.Info("Sign", "type", "response", "data", common.Bytes2Hex(b), "error", e)
+	return b, e
+}
+
+func (l *AuditLogger) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) {
+	l.log.Info("EcRecover", "type", "request", "metadata", MetadataFromContext(ctx).String(),
+		"data", common.Bytes2Hex(data))
+	a, e := l.api.EcRecover(ctx, data, sig)
+	l.log.Info("EcRecover", "type", "response", "addr", a.String(), "error", e)
+	return a, e
+}
+
+func (l *AuditLogger) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
+	l.log.Info("Export", "type", "request", "metadata", MetadataFromContext(ctx).String(),
+		"addr", addr.Hex())
+	j, e := l.api.Export(ctx, addr)
+	// In this case, we don't actually log the json-response, which may be extra sensitive
+	l.log.Info("Export", "type", "response", "json response size", len(j), "error", e)
+	return j, e
+}
+
+func (l *AuditLogger) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) {
+	// Don't actually log the json contents
+	l.log.Info("Import", "type", "request", "metadata", MetadataFromContext(ctx).String(),
+		"keyJSON size", len(keyJSON))
+	a, e := l.api.Import(ctx, keyJSON)
+	l.log.Info("Import", "type", "response", "addr", a.String(), "error", e)
+	return a, e
+}
+
+func NewAuditLogger(path string, api ExternalAPI) (*AuditLogger, error) {
+	l := log.New("api", "signer")
+	handler, err := log.FileHandler(path, log.LogfmtFormat())
+	if err != nil {
+		return nil, err
+	}
+	l.SetHandler(handler)
+	l.Info("Configured", "audit log", path)
+	return &AuditLogger{l, api}, nil
+}

signer/core/cliui.go

@@ -0,0 +1,247 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+package core
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"sync"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+type CommandlineUI struct {
+	in *bufio.Reader
+	mu sync.Mutex
+}
+
+func NewCommandlineUI() *CommandlineUI {
+	return &CommandlineUI{in: bufio.NewReader(os.Stdin)}
+}
+
+// readString reads a single line from stdin, trimming if from spaces, enforcing
+// non-emptyness.
+func (ui *CommandlineUI) readString() string {
+	for {
+		fmt.Printf("> ")
+		text, err := ui.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+	}
+}
+
+// readPassword reads a single line from stdin, trimming it from the trailing new
+// line and returns it. The input will not be echoed.
+func (ui *CommandlineUI) readPassword() string {
+	fmt.Printf("Enter password to approve:\n")
+	fmt.Printf("> ")
+
+	text, err := terminal.ReadPassword(int(os.Stdin.Fd()))
+	if err != nil {
+		log.Crit("Failed to read password", "err", err)
+	}
+	fmt.Println()
+	fmt.Println("-----------------------")
+	return string(text)
+}
+
+// readPassword reads a single line from stdin, trimming it from the trailing new
+// line and returns it. The input will not be echoed.
+func (ui *CommandlineUI) readPasswordText(inputstring string) string {
+	fmt.Printf("Enter %s:\n", inputstring)
+	fmt.Printf("> ")
+	text, err := terminal.ReadPassword(int(os.Stdin.Fd()))
+	if err != nil {
+		log.Crit("Failed to read password", "err", err)
+	}
+	fmt.Println("-----------------------")
+	return string(text)
+}
+
+// confirm returns true if user enters 'Yes', otherwise false
+func (ui *CommandlineUI) confirm() bool {
+	fmt.Printf("Approve? [y/N]:\n")
+	if ui.readString() == "y" {
+		return true
+	}
+	fmt.Println("-----------------------")
+	return false
+}
+
+func showMetadata(metadata Metadata) {
+	fmt.Printf("Request context:\n\t%v -> %v -> %v\n", metadata.Remote, metadata.Scheme, metadata.Local)
+}
+
+// ApproveTx prompt the user for confirmation to request to sign Transaction
+func (ui *CommandlineUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+	weival := request.Transaction.Value.ToInt()
+	fmt.Printf("--------- Transaction request-------------\n")
+	if to := request.Transaction.To; to != nil {
+		fmt.Printf("to:    %v\n", to.Original())
+		if !to.ValidChecksum() {
+			fmt.Printf("\nWARNING: Invalid checksum on to-address!\n\n")
+		}
+	} else {
+		fmt.Printf("to:    <contact creation>\n")
+	}
+	fmt.Printf("from:  %v\n", request.Transaction.From.String())
+	fmt.Printf("value: %v wei\n", weival)
+	if request.Transaction.Data != nil {
+		d := *request.Transaction.Data
+		if len(d) > 0 {
+			fmt.Printf("data:  %v\n", common.Bytes2Hex(d))
+		}
+	}
+	if request.Callinfo != nil {
+		fmt.Printf("\nTransaction validation:\n")
+		for _, m := range request.Callinfo {
+			fmt.Printf("  * %s : %s", m.Typ, m.Message)
+		}
+		fmt.Println()
+
+	}
+	fmt.Printf("\n")
+	showMetadata(request.Meta)
+	fmt.Printf("-------------------------------------------\n")
+	if !ui.confirm() {
+		return SignTxResponse{request.Transaction, false, ""}, nil
+	}
+	return SignTxResponse{request.Transaction, true, ui.readPassword()}, nil
+}
+
+// ApproveSignData prompt the user for confirmation to request to sign data
+func (ui *CommandlineUI) ApproveSignData(request *SignDataRequest) (SignDataResponse, error) {
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+
+	fmt.Printf("-------- Sign data request--------------\n")
+	fmt.Printf("Account:  %s\n", request.Address.String())
+	fmt.Printf("message:  \n%q\n", request.Message)
+	fmt.Printf("raw data: \n%v\n", request.Rawdata)
+	fmt.Printf("message hash:  %v\n", request.Hash)
+	fmt.Printf("-------------------------------------------\n")
+	showMetadata(request.Meta)
+	if !ui.confirm() {
+		return SignDataResponse{false, ""}, nil
+	}
+	return SignDataResponse{true, ui.readPassword()}, nil
+}
+
+// ApproveExport prompt the user for confirmation to export encrypted Account json
+func (ui *CommandlineUI) ApproveExport(request *ExportRequest) (ExportResponse, error) {
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+
+	fmt.Printf("-------- Export Account request--------------\n")
+	fmt.Printf("A request has been made to export the (encrypted) keyfile\n")
+	fmt.Printf("Approving this operation means that the caller obtains the (encrypted) contents\n")
+	fmt.Printf("\n")
+	fmt.Printf("Account:  %x\n", request.Address)
+	//fmt.Printf("keyfile:  \n%v\n", request.file)
+	fmt.Printf("-------------------------------------------\n")
+	showMetadata(request.Meta)
+	return ExportResponse{ui.confirm()}, nil
+}
+
+// ApproveImport prompt the user for confirmation to import Account json
+func (ui *CommandlineUI) ApproveImport(request *ImportRequest) (ImportResponse, error) {
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+
+	fmt.Printf("-------- Import Account request--------------\n")
+	fmt.Printf("A request has been made to import an encrypted keyfile\n")
+	fmt.Printf("-------------------------------------------\n")
+	showMetadata(request.Meta)
+	if !ui.confirm() {
+		return ImportResponse{false, "", ""}, nil
+	}
+	return ImportResponse{true, ui.readPasswordText("Old password"), ui.readPasswordText("New password")}, nil
+}
+
+// ApproveListing prompt the user for confirmation to list accounts
+// the list of accounts to list can be modified by the UI
+func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, error) {
+
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+
+	fmt.Printf("-------- List Account request--------------\n")
+	fmt.Printf("A request has been made to list all accounts. \n")
+	fmt.Printf("You can select which accounts the caller can see\n")
+	for _, account := range request.Accounts {
+		fmt.Printf("\t[x] %v\n", account.Address.Hex())
+	}
+	fmt.Printf("-------------------------------------------\n")
+	showMetadata(request.Meta)
+	if !ui.confirm() {
+		return ListResponse{nil}, nil
+	}
+	return ListResponse{request.Accounts}, nil
+}
+
+// ApproveNewAccount prompt the user for confirmation to create new Account, and reveal to caller
+func (ui *CommandlineUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) {
+
+	ui.mu.Lock()
+	defer ui.mu.Unlock()
+
+	fmt.Printf("-------- New Account request--------------\n")
+	fmt.Printf("A request has been made to create a new. \n")
+	fmt.Printf("Approving this operation means that a new Account is created,\n")
+	fmt.Printf("and the address show to the caller\n")
+	showMetadata(request.Meta)
+	if !ui.confirm() {
+		return NewAccountResponse{false, ""}, nil
+	}
+	return NewAccountResponse{true, ui.readPassword()}, nil
+}
+
+// ShowError displays error message to user
+func (ui *CommandlineUI) ShowError(message string) {
+
+	fmt.Printf("ERROR: %v\n", message)
+}
+
+// ShowInfo displays info message to user
+func (ui *CommandlineUI) ShowInfo(message string) {
+	fmt.Printf("Info: %v\n", message)
+}
+
+func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	fmt.Printf("Transaction signed:\n ")
+	spew.Dump(tx.Tx)
+}
+
+func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) {
+
+	fmt.Printf("------- Signer info -------\n")
+	for k, v := range info.Info {
+		fmt.Printf("* %v : %v\n", k, v)
+	}
+}

signer/core/stdioui.go

@@ -0,0 +1,113 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package core
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+type StdIOUI struct {
+	client rpc.Client
+	mu     sync.Mutex
+}
+
+func NewStdIOUI() *StdIOUI {
+	log.Info("NewStdIOUI")
+	client, err := rpc.DialContext(context.Background(), "stdio://")
+	if err != nil {
+		log.Crit("Could not create stdio client", "err", err)
+	}
+	return &StdIOUI{client: *client}
+}
+
+// dispatch sends a request over the stdio
+func (ui *StdIOUI) dispatch(serviceMethod string, args interface{}, reply interface{}) error {
+	err := ui.client.Call(&reply, serviceMethod, args)
+	if err != nil {
+		log.Info("Error", "exc", err.Error())
+	}
+	return err
+}
+
+func (ui *StdIOUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
+	var result SignTxResponse
+	err := ui.dispatch("ApproveTx", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ApproveSignData(request *SignDataRequest) (SignDataResponse, error) {
+	var result SignDataResponse
+	err := ui.dispatch("ApproveSignData", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ApproveExport(request *ExportRequest) (ExportResponse, error) {
+	var result ExportResponse
+	err := ui.dispatch("ApproveExport", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ApproveImport(request *ImportRequest) (ImportResponse, error) {
+	var result ImportResponse
+	err := ui.dispatch("ApproveImport", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ApproveListing(request *ListRequest) (ListResponse, error) {
+	var result ListResponse
+	err := ui.dispatch("ApproveListing", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) {
+	var result NewAccountResponse
+	err := ui.dispatch("ApproveNewAccount", request, &result)
+	return result, err
+}
+
+func (ui *StdIOUI) ShowError(message string) {
+	err := ui.dispatch("ShowError", &Message{message}, nil)
+	if err != nil {
+		log.Info("Error calling 'ShowError'", "exc", err.Error(), "msg", message)
+	}
+}
+
+func (ui *StdIOUI) ShowInfo(message string) {
+	err := ui.dispatch("ShowInfo", Message{message}, nil)
+	if err != nil {
+		log.Info("Error calling 'ShowInfo'", "exc", err.Error(), "msg", message)
+	}
+}
+func (ui *StdIOUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	err := ui.dispatch("OnApprovedTx", tx, nil)
+	if err != nil {
+		log.Info("Error calling 'OnApprovedTx'", "exc", err.Error(), "tx", tx)
+	}
+}
+
+func (ui *StdIOUI) OnSignerStartup(info StartupInfo) {
+	err := ui.dispatch("OnSignerStartup", info, nil)
+	if err != nil {
+		log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info)
+	}
+}

signer/core/types.go

@@ -0,0 +1,95 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"encoding/json"
+	"strings"
+
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type Accounts []Account
+
+func (as Accounts) String() string {
+	var output []string
+	for _, a := range as {
+		output = append(output, a.String())
+	}
+	return strings.Join(output, "\n")
+}
+
+type Account struct {
+	Typ     string         `json:"type"`
+	URL     accounts.URL   `json:"url"`
+	Address common.Address `json:"address"`
+}
+
+func (a Account) String() string {
+	s, err := json.Marshal(a)
+	if err == nil {
+		return string(s)
+	}
+	return err.Error()
+}
+
+type ValidationInfo struct {
+	Typ     string `json:"type"`
+	Message string `json:"message"`
+}
+type ValidationMessages struct {
+	Messages []ValidationInfo
+}
+
+// SendTxArgs represents the arguments to submit a transaction
+type SendTxArgs struct {
+	From     common.MixedcaseAddress  `json:"from"`
+	To       *common.MixedcaseAddress `json:"to"`
+	Gas      hexutil.Uint64           `json:"gas"`
+	GasPrice hexutil.Big              `json:"gasPrice"`
+	Value    hexutil.Big              `json:"value"`
+	Nonce    hexutil.Uint64           `json:"nonce"`
+	// We accept "data" and "input" for backwards-compatibility reasons.
+	Data  *hexutil.Bytes `json:"data"`
+	Input *hexutil.Bytes `json:"input"`
+}
+
+func (t SendTxArgs) String() string {
+	s, err := json.Marshal(t)
+	if err == nil {
+		return string(s)
+	}
+	return err.Error()
+}
+
+func (args *SendTxArgs) toTransaction() *types.Transaction {
+	var input []byte
+	if args.Data != nil {
+		input = *args.Data
+	} else if args.Input != nil {
+		input = *args.Input
+	}
+	if args.To == nil {
+		return types.NewContractCreation(uint64(args.Nonce), (*big.Int)(&args.Value), uint64(args.Gas), (*big.Int)(&args.GasPrice), input)
+	}
+	return types.NewTransaction(uint64(args.Nonce), args.To.Address(), (*big.Int)(&args.Value), (uint64)(args.Gas), (*big.Int)(&args.GasPrice), input)
+}

signer/core/validation.go

@@ -0,0 +1,163 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// The validation package contains validation checks for transactions
+// - ABI-data validation
+// - Transaction semantics validation
+// The package provides warnings for typical pitfalls
+
+func (vs *ValidationMessages) crit(msg string) {
+	vs.Messages = append(vs.Messages, ValidationInfo{"CRITICAL", msg})
+}
+func (vs *ValidationMessages) warn(msg string) {
+	vs.Messages = append(vs.Messages, ValidationInfo{"WARNING", msg})
+}
+func (vs *ValidationMessages) info(msg string) {
+	vs.Messages = append(vs.Messages, ValidationInfo{"Info", msg})
+}
+
+type Validator struct {
+	db *AbiDb
+}
+
+func NewValidator(db *AbiDb) *Validator {
+	return &Validator{db}
+}
+func testSelector(selector string, data []byte) (*decodedCallData, error) {
+	if selector == "" {
+		return nil, fmt.Errorf("selector not found")
+	}
+	abiData, err := MethodSelectorToAbi(selector)
+	if err != nil {
+		return nil, err
+	}
+	info, err := parseCallData(data, string(abiData))
+	if err != nil {
+		return nil, err
+	}
+	return info, nil
+
+}
+
+// validateCallData checks if the ABI-data + methodselector (if given) can be parsed and seems to match
+func (v *Validator) validateCallData(msgs *ValidationMessages, data []byte, methodSelector *string) {
+	if len(data) == 0 {
+		return
+	}
+	if len(data) < 4 {
+		msgs.warn("Tx contains data which is not valid ABI")
+		return
+	}
+	var (
+		info *decodedCallData
+		err  error
+	)
+	// Check the provided one
+	if methodSelector != nil {
+		info, err = testSelector(*methodSelector, data)
+		if err != nil {
+			msgs.warn(fmt.Sprintf("Tx contains data, but provided ABI signature could not be matched: %v", err))
+		} else {
+			msgs.info(info.String())
+			//Successfull match. add to db if not there already (ignore errors there)
+			v.db.AddSignature(*methodSelector, data[:4])
+		}
+		return
+	}
+	// Check the db
+	selector, err := v.db.LookupMethodSelector(data[:4])
+	if err != nil {
+		msgs.warn(fmt.Sprintf("Tx contains data, but the ABI signature could not be found: %v", err))
+		return
+	}
+	info, err = testSelector(selector, data)
+	if err != nil {
+		msgs.warn(fmt.Sprintf("Tx contains data, but provided ABI signature could not be matched: %v", err))
+	} else {
+		msgs.info(info.String())
+	}
+}
+
+// validateSemantics checks if the transactions 'makes sense', and generate warnings for a couple of typical scenarios
+func (v *Validator) validate(msgs *ValidationMessages, txargs *SendTxArgs, methodSelector *string) error {
+	// Prevent accidental erroneous usage of both 'input' and 'data'
+	if txargs.Data != nil && txargs.Input != nil && !bytes.Equal(*txargs.Data, *txargs.Input) {
+		// This is a showstopper
+		return errors.New(`Ambiguous request: both "data" and "input" are set and are not identical`)
+	}
+	var (
+		data []byte
+	)
+	// Place data on 'data', and nil 'input'
+	if txargs.Input != nil {
+		txargs.Data = txargs.Input
+		txargs.Input = nil
+	}
+	if txargs.Data != nil {
+		data = *txargs.Data
+	}
+
+	if txargs.To == nil {
+		//Contract creation should contain sufficient data to deploy a contract
+		// A typical error is omitting sender due to some quirk in the javascript call
+		// e.g. https://github.com/ethereum/go-ethereum/issues/16106
+		if len(data) == 0 {
+			if txargs.Value.ToInt().Cmp(big.NewInt(0)) > 0 {
+				// Sending ether into black hole
+				return errors.New(`Tx will create contract with value but empty code!`)
+			}
+			// No value submitted at least
+			msgs.crit("Tx will create contract with empty code!")
+		} else if len(data) < 40 { //Arbitrary limit
+			msgs.warn(fmt.Sprintf("Tx will will create contract, but payload is suspiciously small (%d b)", len(data)))
+		}
+		// methodSelector should be nil for contract creation
+		if methodSelector != nil {
+			msgs.warn("Tx will create contract, but method selector supplied; indicating intent to call a method.")
+		}
+
+	} else {
+		if !txargs.To.ValidChecksum() {
+			msgs.warn("Invalid checksum on to-address")
+		}
+		// Normal transaction
+		if bytes.Equal(txargs.To.Address().Bytes(), common.Address{}.Bytes()) {
+			// Sending to 0
+			msgs.crit("Tx destination is the zero address!")
+		}
+		// Validate calldata
+		v.validateCallData(msgs, data, methodSelector)
+	}
+	return nil
+}
+
+// ValidateTransaction does a number of checks on the supplied transaction, and returns either a list of warnings,
+// or an error, indicating that the transaction should be immediately rejected
+func (v *Validator) ValidateTransaction(txArgs *SendTxArgs, methodSelector *string) (*ValidationMessages, error) {
+	msgs := &ValidationMessages{}
+	return msgs, v.validate(msgs, txArgs, methodSelector)
+}

signer/core/validation_test.go

@@ -0,0 +1,139 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"fmt"
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+func hexAddr(a string) common.Address { return common.BytesToAddress(common.FromHex(a)) }
+func mixAddr(a string) (*common.MixedcaseAddress, error) {
+	return common.NewMixedcaseAddressFromString(a)
+}
+func toHexBig(h string) hexutil.Big {
+	b := big.NewInt(0).SetBytes(common.FromHex(h))
+	return hexutil.Big(*b)
+}
+func toHexUint(h string) hexutil.Uint64 {
+	b := big.NewInt(0).SetBytes(common.FromHex(h))
+	return hexutil.Uint64(b.Uint64())
+}
+func dummyTxArgs(t txtestcase) *SendTxArgs {
+	to, _ := mixAddr(t.to)
+	from, _ := mixAddr(t.from)
+	n := toHexUint(t.n)
+	gas := toHexUint(t.g)
+	gasPrice := toHexBig(t.gp)
+	value := toHexBig(t.value)
+	var (
+		data, input *hexutil.Bytes
+	)
+	if t.d != "" {
+		a := hexutil.Bytes(common.FromHex(t.d))
+		data = &a
+	}
+	if t.i != "" {
+		a := hexutil.Bytes(common.FromHex(t.i))
+		input = &a
+
+	}
+	return &SendTxArgs{
+		From:     *from,
+		To:       to,
+		Value:    value,
+		Nonce:    n,
+		GasPrice: gasPrice,
+		Gas:      gas,
+		Data:     data,
+		Input:    input,
+	}
+}
+
+type txtestcase struct {
+	from, to, n, g, gp, value, d, i string
+	expectErr                       bool
+	numMessages                     int
+}
+
+func TestValidator(t *testing.T) {
+	var (
+		// use empty db, there are other tests for the abi-specific stuff
+		db, _ = NewEmptyAbiDB()
+		v     = NewValidator(db)
+	)
+	testcases := []txtestcase{
+		// Invalid to checksum
+		{from: "000000000000000000000000000000000000dead", to: "000000000000000000000000000000000000dead",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", numMessages: 1},
+		// valid 0x000000000000000000000000000000000000dEaD
+		{from: "000000000000000000000000000000000000dead", to: "0x000000000000000000000000000000000000dEaD",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", numMessages: 0},
+		// conflicting input and data
+		{from: "000000000000000000000000000000000000dead", to: "0x000000000000000000000000000000000000dEaD",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", d: "0x01", i: "0x02", expectErr: true},
+		// Data can't be parsed
+		{from: "000000000000000000000000000000000000dead", to: "0x000000000000000000000000000000000000dEaD",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", d: "0x0102", numMessages: 1},
+		// Data (on Input) can't be parsed
+		{from: "000000000000000000000000000000000000dead", to: "0x000000000000000000000000000000000000dEaD",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", i: "0x0102", numMessages: 1},
+		// Send to 0
+		{from: "000000000000000000000000000000000000dead", to: "0x0000000000000000000000000000000000000000",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", numMessages: 1},
+		// Create empty contract (no value)
+		{from: "000000000000000000000000000000000000dead", to: "",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x00", numMessages: 1},
+		// Create empty contract (with value)
+		{from: "000000000000000000000000000000000000dead", to: "",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", expectErr: true},
+		// Small payload for create
+		{from: "000000000000000000000000000000000000dead", to: "",
+			n: "0x01", g: "0x20", gp: "0x40", value: "0x01", d: "0x01", numMessages: 1},
+	}
+	for i, test := range testcases {
+		msgs, err := v.ValidateTransaction(dummyTxArgs(test), nil)
+		if err == nil && test.expectErr {
+			t.Errorf("Test %d, expected error", i)
+			for _, msg := range msgs.Messages {
+				fmt.Printf("* %s: %s\n", msg.Typ, msg.Message)
+			}
+		}
+		if err != nil && !test.expectErr {
+			t.Errorf("Test %d, unexpected error: %v", i, err)
+		}
+		if err == nil {
+			got := len(msgs.Messages)
+			if got != test.numMessages {
+				for _, msg := range msgs.Messages {
+					fmt.Printf("* %s: %s\n", msg.Typ, msg.Message)
+				}
+				t.Errorf("Test %d, expected %d messages, got %d", i, test.numMessages, got)
+			} else {
+				//Debug printout, remove later
+				for _, msg := range msgs.Messages {
+					fmt.Printf("* [%d] %s: %s\n", i, msg.Typ, msg.Message)
+				}
+				fmt.Println()
+			}
+		}
+	}
+}

signer/rules/deps/deps.go

@@ -0,0 +1,21 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package deps contains the console JavaScript dependencies Go embedded.
+package deps
+
+//go:generate go-bindata -nometadata -pkg deps -o bindata.go bignumber.js
+//go:generate gofmt -w -s bindata.go

signer/rules/rules.go

@@ -0,0 +1,248 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package rules
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/rules/deps"
+	"github.com/ethereum/go-ethereum/signer/storage"
+	"github.com/robertkrimen/otto"
+)
+
+var (
+	BigNumber_JS = deps.MustAsset("bignumber.js")
+)
+
+// consoleOutput is an override for the console.log and console.error methods to
+// stream the output into the configured output stream instead of stdout.
+func consoleOutput(call otto.FunctionCall) otto.Value {
+	output := []string{"JS:> "}
+	for _, argument := range call.ArgumentList {
+		output = append(output, fmt.Sprintf("%v", argument))
+	}
+	fmt.Fprintln(os.Stdout, strings.Join(output, " "))
+	return otto.Value{}
+}
+
+// rulesetUi provides an implementation of SignerUI that evaluates a javascript
+// file for each defined UI-method
+type rulesetUi struct {
+	next        core.SignerUI // The next handler, for manual processing
+	storage     storage.Storage
+	credentials storage.Storage
+	jsRules     string // The rules to use
+}
+
+func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUi, error) {
+	c := &rulesetUi{
+		next:        next,
+		storage:     jsbackend,
+		credentials: credentialsBackend,
+		jsRules:     "",
+	}
+
+	return c, nil
+}
+
+func (r *rulesetUi) Init(javascriptRules string) error {
+	r.jsRules = javascriptRules
+	return nil
+}
+func (r *rulesetUi) execute(jsfunc string, jsarg interface{}) (otto.Value, error) {
+
+	// Instantiate a fresh vm engine every time
+	vm := otto.New()
+	// Set the native callbacks
+	consoleObj, _ := vm.Get("console")
+	consoleObj.Object().Set("log", consoleOutput)
+	consoleObj.Object().Set("error", consoleOutput)
+	vm.Set("storage", r.storage)
+
+	// Load bootstrap libraries
+	script, err := vm.Compile("bignumber.js", BigNumber_JS)
+	if err != nil {
+		log.Warn("Failed loading libraries", "err", err)
+		return otto.UndefinedValue(), err
+	}
+	vm.Run(script)
+
+	// Run the actual rule implementation
+	_, err = vm.Run(r.jsRules)
+	if err != nil {
+		log.Warn("Execution failed", "err", err)
+		return otto.UndefinedValue(), err
+	}
+
+	// And the actual call
+	// All calls are objects with the parameters being keys in that object.
+	// To provide additional insulation between js and go, we serialize it into JSON on the Go-side,
+	// and deserialize it on the JS side.
+
+	jsonbytes, err := json.Marshal(jsarg)
+	if err != nil {
+		log.Warn("failed marshalling data", "data", jsarg)
+		return otto.UndefinedValue(), err
+	}
+	// Now, we call foobar(JSON.parse(<jsondata>)).
+	var call string
+	if len(jsonbytes) > 0 {
+		call = fmt.Sprintf("%v(JSON.parse(%v))", jsfunc, string(jsonbytes))
+	} else {
+		call = fmt.Sprintf("%v()", jsfunc)
+	}
+	return vm.Run(call)
+}
+
+func (r *rulesetUi) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) {
+	if err != nil {
+		return false, err
+	}
+	v, err := r.execute(jsfunc, string(jsarg))
+	if err != nil {
+		log.Info("error occurred during execution", "error", err)
+		return false, err
+	}
+	result, err := v.ToString()
+	if err != nil {
+		log.Info("error occurred during response unmarshalling", "error", err)
+		return false, err
+	}
+	if result == "Approve" {
+		log.Info("Op approved")
+		return true, nil
+	} else if result == "Reject" {
+		log.Info("Op rejected")
+		return false, nil
+	}
+	return false, fmt.Errorf("Unknown response")
+}
+
+func (r *rulesetUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
+	jsonreq, err := json.Marshal(request)
+	approved, err := r.checkApproval("ApproveTx", jsonreq, err)
+	if err != nil {
+		log.Info("Rule-based approval error, going to manual", "error", err)
+		return r.next.ApproveTx(request)
+	}
+
+	if approved {
+		return core.SignTxResponse{
+				Transaction: request.Transaction,
+				Approved:    true,
+				Password:    r.lookupPassword(request.Transaction.From.Address()),
+			},
+			nil
+	}
+	return core.SignTxResponse{Approved: false}, err
+}
+
+func (r *rulesetUi) lookupPassword(address common.Address) string {
+	return r.credentials.Get(strings.ToLower(address.String()))
+}
+
+func (r *rulesetUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) {
+	jsonreq, err := json.Marshal(request)
+	approved, err := r.checkApproval("ApproveSignData", jsonreq, err)
+	if err != nil {
+		log.Info("Rule-based approval error, going to manual", "error", err)
+		return r.next.ApproveSignData(request)
+	}
+	if approved {
+		return core.SignDataResponse{Approved: true, Password: r.lookupPassword(request.Address.Address())}, nil
+	}
+	return core.SignDataResponse{Approved: false, Password: ""}, err
+}
+
+func (r *rulesetUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) {
+	jsonreq, err := json.Marshal(request)
+	approved, err := r.checkApproval("ApproveExport", jsonreq, err)
+	if err != nil {
+		log.Info("Rule-based approval error, going to manual", "error", err)
+		return r.next.ApproveExport(request)
+	}
+	if approved {
+		return core.ExportResponse{Approved: true}, nil
+	}
+	return core.ExportResponse{Approved: false}, err
+}
+
+func (r *rulesetUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) {
+	// This cannot be handled by rules, requires setting a password
+	// dispatch to next
+	return r.next.ApproveImport(request)
+}
+
+func (r *rulesetUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) {
+	jsonreq, err := json.Marshal(request)
+	approved, err := r.checkApproval("ApproveListing", jsonreq, err)
+	if err != nil {
+		log.Info("Rule-based approval error, going to manual", "error", err)
+		return r.next.ApproveListing(request)
+	}
+	if approved {
+		return core.ListResponse{Accounts: request.Accounts}, nil
+	}
+	return core.ListResponse{}, err
+}
+
+func (r *rulesetUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) {
+	// This cannot be handled by rules, requires setting a password
+	// dispatch to next
+	return r.next.ApproveNewAccount(request)
+}
+
+func (r *rulesetUi) ShowError(message string) {
+	log.Error(message)
+	r.next.ShowError(message)
+}
+
+func (r *rulesetUi) ShowInfo(message string) {
+	log.Info(message)
+	r.next.ShowInfo(message)
+}
+func (r *rulesetUi) OnSignerStartup(info core.StartupInfo) {
+	jsonInfo, err := json.Marshal(info)
+	if err != nil {
+		log.Warn("failed marshalling data", "data", info)
+		return
+	}
+	r.next.OnSignerStartup(info)
+	_, err = r.execute("OnSignerStartup", string(jsonInfo))
+	if err != nil {
+		log.Info("error occurred during execution", "error", err)
+	}
+}
+
+func (r *rulesetUi) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	jsonTx, err := json.Marshal(tx)
+	if err != nil {
+		log.Warn("failed marshalling transaction", "tx", tx)
+		return
+	}
+	_, err = r.execute("OnApprovedTx", string(jsonTx))
+	if err != nil {
+		log.Info("error occurred during execution", "error", err)
+	}
+}

signer/rules/rules_test.go

@@ -0,0 +1,631 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package rules
+
+import (
+	"fmt"
+	"math/big"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/storage"
+)
+
+const JS = `
+/**
+This is an example implementation of a Javascript rule file. 
+
+When the signer receives a request over the external API, the corresponding method is evaluated. 
+Three things can happen: 
+
+1. The method returns "Approve". This means the operation is permitted. 
+2. The method returns "Reject". This means the operation is rejected. 
+3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means
+that the operation will continue to manual processing, via the regular UI method chosen by the user. 
+
+[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not 
+only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all 
+accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject").
+
+**/
+
+function ApproveListing(request){
+	console.log("In js approve listing");
+	console.log(request.accounts[3].Address)
+	console.log(request.meta.Remote)
+	return "Approve"
+}
+
+function ApproveTx(request){
+	console.log("test");
+	console.log("from");
+	return "Reject";
+}
+
+function test(thing){
+	console.log(thing.String())
+}
+
+`
+
+func mixAddr(a string) (*common.MixedcaseAddress, error) {
+	return common.NewMixedcaseAddressFromString(a)
+}
+
+type alwaysDenyUi struct{}
+
+func (alwaysDenyUi) OnSignerStartup(info core.StartupInfo) {
+}
+
+func (alwaysDenyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
+	return core.SignTxResponse{Transaction: request.Transaction, Approved: false, Password: ""}, nil
+}
+
+func (alwaysDenyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) {
+	return core.SignDataResponse{Approved: false, Password: ""}, nil
+}
+
+func (alwaysDenyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) {
+	return core.ExportResponse{Approved: false}, nil
+}
+
+func (alwaysDenyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) {
+	return core.ImportResponse{Approved: false, OldPassword: "", NewPassword: ""}, nil
+}
+
+func (alwaysDenyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) {
+	return core.ListResponse{Accounts: nil}, nil
+}
+
+func (alwaysDenyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) {
+	return core.NewAccountResponse{Approved: false, Password: ""}, nil
+}
+
+func (alwaysDenyUi) ShowError(message string) {
+	panic("implement me")
+}
+
+func (alwaysDenyUi) ShowInfo(message string) {
+	panic("implement me")
+}
+
+func (alwaysDenyUi) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	panic("implement me")
+}
+
+func initRuleEngine(js string) (*rulesetUi, error) {
+	r, err := NewRuleEvaluator(&alwaysDenyUi{}, storage.NewEphemeralStorage(), storage.NewEphemeralStorage())
+	if err != nil {
+		return nil, fmt.Errorf("failed to create js engine: %v", err)
+	}
+	if err = r.Init(js); err != nil {
+		return nil, fmt.Errorf("failed to load bootstrap js: %v", err)
+	}
+	return r, nil
+}
+
+func TestListRequest(t *testing.T) {
+	accs := make([]core.Account, 5)
+
+	for i := range accs {
+		addr := fmt.Sprintf("000000000000000000000000000000000000000%x", i)
+		acc := core.Account{
+			Address: common.BytesToAddress(common.Hex2Bytes(addr)),
+			URL:     accounts.URL{Scheme: "test", Path: fmt.Sprintf("acc-%d", i)},
+		}
+		accs[i] = acc
+	}
+
+	js := `function ApproveListing(){ return "Approve" }`
+
+	r, err := initRuleEngine(js)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+	resp, err := r.ApproveListing(&core.ListRequest{
+		Accounts: accs,
+		Meta:     core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
+	})
+	if len(resp.Accounts) != len(accs) {
+		t.Errorf("Expected check to resolve to 'Approve'")
+	}
+}
+
+func TestSignTxRequest(t *testing.T) {
+
+	js := `
+	function ApproveTx(r){
+		console.log("transaction.from", r.transaction.from);
+		console.log("transaction.to", r.transaction.to);
+		console.log("transaction.value", r.transaction.value);
+		console.log("transaction.nonce", r.transaction.nonce);
+		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
+	}`
+
+	r, err := initRuleEngine(js)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+	to, err := mixAddr("000000000000000000000000000000000000dead")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	from, err := mixAddr("0000000000000000000000000000000000001337")
+
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	fmt.Printf("to %v", to.Address().String())
+	resp, err := r.ApproveTx(&core.SignTxRequest{
+		Transaction: core.SendTxArgs{
+			From: *from,
+			To:   to},
+		Callinfo: nil,
+		Meta:     core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
+	})
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	if !resp.Approved {
+		t.Errorf("Expected check to resolve to 'Approve'")
+	}
+}
+
+type dummyUi struct {
+	calls []string
+}
+
+func (d *dummyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
+	d.calls = append(d.calls, "ApproveTx")
+	return core.SignTxResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) {
+	d.calls = append(d.calls, "ApproveSignData")
+	return core.SignDataResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) {
+	d.calls = append(d.calls, "ApproveExport")
+	return core.ExportResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) {
+	d.calls = append(d.calls, "ApproveImport")
+	return core.ImportResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) {
+	d.calls = append(d.calls, "ApproveListing")
+	return core.ListResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) {
+	d.calls = append(d.calls, "ApproveNewAccount")
+	return core.NewAccountResponse{}, core.ErrRequestDenied
+}
+
+func (d *dummyUi) ShowError(message string) {
+	d.calls = append(d.calls, "ShowError")
+}
+
+func (d *dummyUi) ShowInfo(message string) {
+	d.calls = append(d.calls, "ShowInfo")
+}
+
+func (d *dummyUi) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	d.calls = append(d.calls, "OnApprovedTx")
+}
+func (d *dummyUi) OnSignerStartup(info core.StartupInfo) {
+}
+
+//TestForwarding tests that the rule-engine correctly dispatches requests to the next caller
+func TestForwarding(t *testing.T) {
+
+	js := ""
+	ui := &dummyUi{make([]string, 0)}
+	jsBackend := storage.NewEphemeralStorage()
+	credBackend := storage.NewEphemeralStorage()
+	r, err := NewRuleEvaluator(ui, jsBackend, credBackend)
+	if err != nil {
+		t.Fatalf("Failed to create js engine: %v", err)
+	}
+	if err = r.Init(js); err != nil {
+		t.Fatalf("Failed to load bootstrap js: %v", err)
+	}
+	r.ApproveSignData(nil)
+	r.ApproveTx(nil)
+	r.ApproveImport(nil)
+	r.ApproveNewAccount(nil)
+	r.ApproveListing(nil)
+	r.ApproveExport(nil)
+	r.ShowError("test")
+	r.ShowInfo("test")
+
+	//This one is not forwarded
+	r.OnApprovedTx(ethapi.SignTransactionResult{})
+
+	expCalls := 8
+	if len(ui.calls) != expCalls {
+
+		t.Errorf("Expected %d forwarded calls, got %d: %s", expCalls, len(ui.calls), strings.Join(ui.calls, ","))
+
+	}
+
+}
+
+func TestMissingFunc(t *testing.T) {
+	r, err := initRuleEngine(JS)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+
+	_, err = r.execute("MissingMethod", "test")
+
+	if err == nil {
+		t.Error("Expected error")
+	}
+
+	approved, err := r.checkApproval("MissingMethod", nil, nil)
+	if err == nil {
+		t.Errorf("Expected missing method to yield error'")
+	}
+	if approved {
+		t.Errorf("Expected missing method to cause non-approval")
+	}
+	fmt.Printf("Err %v", err)
+
+}
+func TestStorage(t *testing.T) {
+
+	js := `
+	function testStorage(){
+		storage.Put("mykey", "myvalue")
+		a = storage.Get("mykey")
+		
+		storage.Put("mykey", ["a", "list"])  	// Should result in "a,list"
+		a += storage.Get("mykey")
+
+		
+		storage.Put("mykey", {"an": "object"}) 	// Should result in "[object Object]"
+		a += storage.Get("mykey")
+
+		
+		storage.Put("mykey", JSON.stringify({"an": "object"})) // Should result in '{"an":"object"}'
+		a += storage.Get("mykey")
+
+		a += storage.Get("missingkey")		//Missing keys should result in empty string
+		storage.Put("","missing key==noop") // Can't store with 0-length key
+		a += storage.Get("")				// Should result in ''
+		
+		var b = new BigNumber(2)
+		var c = new BigNumber(16)//"0xf0",16)
+		var d = b.plus(c)
+		console.log(d)
+		return a
+	}
+`
+	r, err := initRuleEngine(js)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+
+	v, err := r.execute("testStorage", nil)
+
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+
+	retval, err := v.ToString()
+
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	exp := `myvaluea,list[object Object]{"an":"object"}`
+	if retval != exp {
+		t.Errorf("Unexpected data, expected '%v', got '%v'", exp, retval)
+	}
+	fmt.Printf("Err %v", err)
+
+}
+
+const ExampleTxWindow = `
+	function big(str){
+		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str)
+	}
+	
+	// Time window: 1 week
+	var window = 1000* 3600*24*7;
+
+	// Limit : 1 ether
+	var limit = new BigNumber("1e18");
+
+	function isLimitOk(transaction){
+		var value = big(transaction.value)
+		// Start of our window function		
+		var windowstart = new Date().getTime() - window;
+
+		var txs = [];
+		var stored = storage.Get('txs');
+
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// First, remove all that have passed out of the time-window
+		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
+		console.log(txs, newtxs.length);
+	
+		// Secondly, aggregate the current sum
+		sum = new BigNumber(0)
+
+		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+		console.log("ApproveTx > Sum so far", sum);
+		console.log("ApproveTx > Requested", value.toNumber());
+		
+		// Would we exceed weekly limit ?
+		return sum.plus(value).lt(limit)
+		
+	}
+	function ApproveTx(r){
+		console.log(r)
+		console.log(typeof(r))
+		if (isLimitOk(r.transaction)){
+			return "Approve"
+		}
+		return "Nope"
+	}
+
+	/**
+	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+ 	* 'response_str' contains the return value that will be sent to the external caller. 
+	* The return value from this method is ignore - the reason for having this callback is to allow the 
+	* ruleset to keep track of approved transactions. 
+	*
+	* When implementing rate-limited rules, this callback should be used. 
+	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+	* then accepts the transaction, this method will be called.
+	* 
+	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+	*/
+ 	function OnApprovedTx(resp){
+		var value = big(resp.tx.value)
+		var txs = []
+		// Load stored transactions
+		var stored = storage.Get('txs');
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// Add this to the storage
+		txs.push({tstamp: new Date().getTime(), value: value});
+		storage.Put("txs", JSON.stringify(txs));
+	}
+
+`
+
+func dummyTx(value hexutil.Big) *core.SignTxRequest {
+
+	to, _ := mixAddr("000000000000000000000000000000000000dead")
+	from, _ := mixAddr("000000000000000000000000000000000000dead")
+	n := hexutil.Uint64(3)
+	gas := hexutil.Uint64(21000)
+	gasPrice := hexutil.Big(*big.NewInt(2000000))
+
+	return &core.SignTxRequest{
+		Transaction: core.SendTxArgs{
+			From:     *from,
+			To:       to,
+			Value:    value,
+			Nonce:    n,
+			GasPrice: gasPrice,
+			Gas:      gas,
+		},
+		Callinfo: []core.ValidationInfo{
+			{Typ: "Warning", Message: "All your base are bellong to us"},
+		},
+		Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
+	}
+}
+func dummyTxWithV(value uint64) *core.SignTxRequest {
+
+	v := big.NewInt(0).SetUint64(value)
+	h := hexutil.Big(*v)
+	return dummyTx(h)
+}
+func dummySigned(value *big.Int) *types.Transaction {
+	to := common.HexToAddress("000000000000000000000000000000000000dead")
+	gas := uint64(21000)
+	gasPrice := big.NewInt(2000000)
+	data := make([]byte, 0)
+	return types.NewTransaction(3, to, value, gas, gasPrice, data)
+
+}
+func TestLimitWindow(t *testing.T) {
+
+	r, err := initRuleEngine(ExampleTxWindow)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+
+	// 0.3 ether: 429D069189E0000 wei
+	v := big.NewInt(0).SetBytes(common.Hex2Bytes("0429D069189E0000"))
+	h := hexutil.Big(*v)
+	// The first three should succeed
+	for i := 0; i < 3; i++ {
+		unsigned := dummyTx(h)
+		resp, err := r.ApproveTx(unsigned)
+		if err != nil {
+			t.Errorf("Unexpected error %v", err)
+		}
+		if !resp.Approved {
+			t.Errorf("Expected check to resolve to 'Approve'")
+		}
+		// Create a dummy signed transaction
+
+		response := ethapi.SignTransactionResult{
+			Tx:  dummySigned(v),
+			Raw: common.Hex2Bytes("deadbeef"),
+		}
+		r.OnApprovedTx(response)
+	}
+	// Fourth should fail
+	resp, err := r.ApproveTx(dummyTx(h))
+	if resp.Approved {
+		t.Errorf("Expected check to resolve to 'Reject'")
+	}
+
+}
+
+// dontCallMe is used as a next-handler that does not want to be called - it invokes test failure
+type dontCallMe struct {
+	t *testing.T
+}
+
+func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
+}
+
+func (d *dontCallMe) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.SignTxResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.SignDataResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.ExportResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.ImportResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ApproveListing(request *core.ListRequest) (core.ListResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.ListResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+	return core.NewAccountResponse{}, core.ErrRequestDenied
+}
+
+func (d *dontCallMe) ShowError(message string) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+}
+
+func (d *dontCallMe) ShowInfo(message string) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+}
+
+func (d *dontCallMe) OnApprovedTx(tx ethapi.SignTransactionResult) {
+	d.t.Fatalf("Did not expect next-handler to be called")
+}
+
+//TestContextIsCleared tests that the rule-engine does not retain variables over several requests.
+// if it does, that would be bad since developers may rely on that to store data,
+// instead of using the disk-based data storage
+func TestContextIsCleared(t *testing.T) {
+
+	js := `
+	function ApproveTx(){
+		if (typeof foobar == 'undefined') {
+			foobar = "Approve"
+ 		}
+		console.log(foobar)
+		if (foobar == "Approve"){
+			foobar = "Reject"
+		}else{
+			foobar = "Approve"
+		}
+		return foobar
+	}
+	`
+	ui := &dontCallMe{t}
+	r, err := NewRuleEvaluator(ui, storage.NewEphemeralStorage(), storage.NewEphemeralStorage())
+	if err != nil {
+		t.Fatalf("Failed to create js engine: %v", err)
+	}
+	if err = r.Init(js); err != nil {
+		t.Fatalf("Failed to load bootstrap js: %v", err)
+	}
+	tx := dummyTxWithV(0)
+	r1, err := r.ApproveTx(tx)
+	r2, err := r.ApproveTx(tx)
+	if r1.Approved != r2.Approved {
+		t.Errorf("Expected execution context to be cleared between executions")
+	}
+}
+
+func TestSignData(t *testing.T) {
+
+	js := `function ApproveListing(){
+    return "Approve"
+}
+function ApproveSignData(r){
+    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+    {
+        if(r.message.indexOf("bazonk") >= 0){
+            return "Approve"
+        }
+        return "Reject"
+    }
+    // Otherwise goes to manual processing
+}`
+	r, err := initRuleEngine(js)
+	if err != nil {
+		t.Errorf("Couldn't create evaluator %v", err)
+		return
+	}
+	message := []byte("baz bazonk foo")
+	hash, msg := core.SignHash(message)
+	raw := hexutil.Bytes(message)
+	addr, _ := mixAddr("0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+
+	fmt.Printf("address %v %v\n", addr.String(), addr.Original())
+	resp, err := r.ApproveSignData(&core.SignDataRequest{
+		Address: *addr,
+		Message: msg,
+		Hash:    hash,
+		Meta:    core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
+		Rawdata: raw,
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error %v", err)
+	}
+	if !resp.Approved {
+		t.Fatalf("Expected approved")
+	}
+}

signer/storage/aes_gcm_storage.go

@@ -0,0 +1,164 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package storage
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type storedCredential struct {
+	// The iv
+	Iv []byte `json:"iv"`
+	// The ciphertext
+	CipherText []byte `json:"c"`
+}
+
+// AESEncryptedStorage is a storage type which is backed by a json-faile. The json-file contains
+// key-value mappings, where the keys are _not_ encrypted, only the values are.
+type AESEncryptedStorage struct {
+	// File to read/write credentials
+	filename string
+	// Key stored in base64
+	key []byte
+}
+
+// NewAESEncryptedStorage creates a new encrypted storage backed by the given file/key
+func NewAESEncryptedStorage(filename string, key []byte) *AESEncryptedStorage {
+	return &AESEncryptedStorage{
+		filename: filename,
+		key:      key,
+	}
+}
+
+// Put stores a value by key. 0-length keys results in no-op
+func (s *AESEncryptedStorage) Put(key, value string) {
+	if len(key) == 0 {
+		return
+	}
+	data, err := s.readEncryptedStorage()
+	if err != nil {
+		log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		return
+	}
+	ciphertext, iv, err := encrypt(s.key, []byte(value))
+	if err != nil {
+		log.Warn("Failed to encrypt entry", "err", err)
+		return
+	}
+	encrypted := storedCredential{Iv: iv, CipherText: ciphertext}
+	data[key] = encrypted
+	if err = s.writeEncryptedStorage(data); err != nil {
+		log.Warn("Failed to write entry", "err", err)
+	}
+}
+
+// Get returns the previously stored value, or the empty string if it does not exist or key is of 0-length
+func (s *AESEncryptedStorage) Get(key string) string {
+	if len(key) == 0 {
+		return ""
+	}
+	data, err := s.readEncryptedStorage()
+	if err != nil {
+		log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		return ""
+	}
+	encrypted, exist := data[key]
+	if !exist {
+		log.Warn("Key does not exist", "key", key)
+		return ""
+	}
+	entry, err := decrypt(s.key, encrypted.Iv, encrypted.CipherText)
+	if err != nil {
+		log.Warn("Failed to decrypt key", "key", key)
+		return ""
+	}
+	return string(entry)
+}
+
+// readEncryptedStorage reads the file with encrypted creds
+func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]storedCredential, error) {
+	creds := make(map[string]storedCredential)
+	raw, err := ioutil.ReadFile(s.filename)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Doesn't exist yet
+			return creds, nil
+
+		} else {
+			log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		}
+	}
+	if err = json.Unmarshal(raw, &creds); err != nil {
+		log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename)
+		return nil, err
+	}
+	return creds, nil
+}
+
+// writeEncryptedStorage write the file with encrypted creds
+func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCredential) error {
+	raw, err := json.Marshal(creds)
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(s.filename, raw, 0600); err != nil {
+		return err
+	}
+	return nil
+}
+
+func encrypt(key []byte, plaintext []byte) ([]byte, []byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, nil, err
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	nonce := make([]byte, aesgcm.NonceSize())
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, nil, err
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+	ciphertext := aesgcm.Seal(nil, nonce, plaintext, nil)
+	return ciphertext, nonce, nil
+}
+
+func decrypt(key []byte, nonce []byte, ciphertext []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return nil, err
+	}
+	return plaintext, nil
+}

signer/storage/aes_gcm_storage_test.go

@@ -0,0 +1,115 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/mattn/go-colorable"
+)
+
+func TestEncryption(t *testing.T) {
+	//	key := []byte("AES256Key-32Characters1234567890")
+	//	plaintext := []byte(value)
+	key := []byte("AES256Key-32Characters1234567890")
+	plaintext := []byte("exampleplaintext")
+
+	c, iv, err := encrypt(key, plaintext)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Printf("Ciphertext %x, nonce %x\n", c, iv)
+
+	p, err := decrypt(key, iv, c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Printf("Plaintext %v\n", string(p))
+	if !bytes.Equal(plaintext, p) {
+		t.Errorf("Failed: expected plaintext recovery, got %v expected %v", string(plaintext), string(p))
+	}
+}
+
+func TestFileStorage(t *testing.T) {
+
+	a := map[string]storedCredential{
+		"secret": {
+			Iv:         common.Hex2Bytes("cdb30036279601aeee60f16b"),
+			CipherText: common.Hex2Bytes("f311ac49859d7260c2c464c28ffac122daf6be801d3cfd3edcbde7e00c9ff74f"),
+		},
+		"secret2": {
+			Iv:         common.Hex2Bytes("afb8a7579bf971db9f8ceeed"),
+			CipherText: common.Hex2Bytes("2df87baf86b5073ef1f03e3cc738de75b511400f5465bb0ddeacf47ae4dc267d"),
+		},
+	}
+	d, err := ioutil.TempDir("", "eth-encrypted-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	stored := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	stored.writeEncryptedStorage(a)
+	read := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	creds, err := read.readEncryptedStorage()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for k, v := range a {
+		if v2, exist := creds[k]; !exist {
+			t.Errorf("Missing entry %v", k)
+		} else {
+			if !bytes.Equal(v.CipherText, v2.CipherText) {
+				t.Errorf("Wrong ciphertext, expected %x got %x", v.CipherText, v2.CipherText)
+			}
+			if !bytes.Equal(v.Iv, v2.Iv) {
+				t.Errorf("Wrong iv")
+			}
+		}
+	}
+}
+func TestEnd2End(t *testing.T) {
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
+	d, err := ioutil.TempDir("", "eth-encrypted-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s1 := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	s2 := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+
+	s1.Put("bazonk", "foobar")
+	if v := s2.Get("bazonk"); v != "foobar" {
+		t.Errorf("Expected bazonk->foobar, got '%v'", v)
+	}
+}

signer/storage/storage.go

@@ -0,0 +1,62 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package storage
+
+import (
+	"fmt"
+)
+
+type Storage interface {
+	// Put stores a value by key. 0-length keys results in no-op
+	Put(key, value string)
+	// Get returns the previously stored value, or the empty string if it does not exist or key is of 0-length
+	Get(key string) string
+}
+
+// EphemeralStorage is an in-memory storage that does
+// not persist values to disk. Mainly used for testing
+type EphemeralStorage struct {
+	data      map[string]string
+	namespace string
+}
+
+func (s *EphemeralStorage) Put(key, value string) {
+	if len(key) == 0 {
+		return
+	}
+	fmt.Printf("storage: put %v -> %v\n", key, value)
+	s.data[key] = value
+}
+
+func (s *EphemeralStorage) Get(key string) string {
+	if len(key) == 0 {
+		return ""
+	}
+	fmt.Printf("storage: get %v\n", key)
+	if v, exist := s.data[key]; exist {
+		return v
+	}
+	return ""
+}
+
+func NewEphemeralStorage() Storage {
+	s := &EphemeralStorage{
+		data: make(map[string]string),
+	}
+	return s
+}

swarm/storage/database.go

@@ -22,7 +22,6 @@ package storage
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/compression/rle"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -31,8 +30,7 @@ import (
 const openFileLimit = 128
 
 type LDBDatabase struct {
-	db   *leveldb.DB
-	comp bool
+	db *leveldb.DB
 }
 
 func NewLDBDatabase(file string) (*LDBDatabase, error) {
@@ -42,16 +40,12 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) {
 		return nil, err
 	}
 
-	database := &LDBDatabase{db: db, comp: false}
+	database := &LDBDatabase{db: db}
 
 	return database, nil
 }
 
 func (self *LDBDatabase) Put(key []byte, value []byte) {
-	if self.comp {
-		value = rle.Compress(value)
-	}
-
 	err := self.db.Put(key, value, nil)
 	if err != nil {
 		fmt.Println("Error put", err)
@@ -63,11 +57,6 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	if self.comp {
-		return rle.Decompress(dat)
-	}
-
 	return dat, nil
 }
 

tests/state_test.go

@@ -36,14 +36,7 @@ func TestState(t *testing.T) {
 	st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet
 	// Expected failures:
 	st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test")
-	st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/EIP158`, "bug in test")
 	st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test")
-	st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/Byzantium`, "bug in test")
-	st.fails(`^stRandom2/randomStatetest64[45]\.json/(EIP150|Frontier|Homestead)/.*`, "known bug #15119")
-	st.fails(`^stCreateTest/TransactionCollisionToEmpty\.json/EIP158/2`, "known bug ")
-	st.fails(`^stCreateTest/TransactionCollisionToEmpty\.json/EIP158/3`, "known bug ")
-	st.fails(`^stCreateTest/TransactionCollisionToEmpty\.json/Byzantium/2`, "known bug ")
-	st.fails(`^stCreateTest/TransactionCollisionToEmpty\.json/Byzantium/3`, "known bug ")
 
 	st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) {
 		for _, subtest := range test.Subtests() {

trie/sync.go

@@ -212,7 +212,7 @@ func (s *TrieSync) Process(results []SyncResult) (bool, int, error) {
 }
 
 // Commit flushes the data stored in the internal membatch out to persistent
-// storage, returning th enumber of items written and any occurred error.
+// storage, returning the number of items written and any occurred error.
 func (s *TrieSync) Commit(dbw ethdb.Putter) (int, error) {
 	// Dump the membatch into a database dbw
 	for i, key := range s.membatch.order {