params: release Geth v1.8.4

* signer: introduce external signer command

* cmd/signer, rpc: Implement new signer. Add info about remote user to Context

* signer: refactored request/response, made use of urfave.cli

* cmd/signer: Use common flags

* cmd/signer: methods to validate calldata against abi

* cmd/signer: work on abi parser

* signer: add mutex around UI

* cmd/signer: add json 4byte directory, remove passwords from api

* cmd/signer: minor changes

* cmd/signer: Use ErrRequestDenied, enable lightkdf

* cmd/signer: implement tests

* cmd/signer: made possible for UI to modify tx parameters

* cmd/signer: refactors, removed channels in ui comms, added UI-api via stdin/out

* cmd/signer: Made lowercase json-definitions, added UI-signer test functionality

* cmd/signer: update documentation

* cmd/signer: fix bugs, improve abi detection, abi argument display

* cmd/signer: minor change in json format

* cmd/signer: rework json communication

* cmd/signer: implement mixcase addresses in API, fix json id bug

* cmd/signer: rename fromaccount, update pythonpoc with new json encoding format

* cmd/signer: make use of new abi interface

* signer: documentation

* signer/main: remove redundant  option

* signer: implement audit logging

* signer: create package 'signer', minor changes

* common: add 0x-prefix to mixcaseaddress in json marshalling + validation

* signer, rules, storage: implement rules + ephemeral storage for signer rules

* signer: implement OnApprovedTx, change signing response (API BREAKAGE)

* signer: refactoring + documentation

* signer/rules: implement dispatching to next handler

* signer: docs

* signer/rules: hide json-conversion from users, ensure context is cleaned

* signer: docs

* signer: implement validation rules, change signature of call_info

* signer: fix log flaw with string pointer

* signer: implement custom 4byte databsae that saves submitted signatures

* signer/storage: implement aes-gcm-backed credential storage

* accounts: implement json unmarshalling of url

* signer: fix listresponse, fix gas->uint64

* node: make http/ipc start methods public

* signer: add ipc capability+review concerns

* accounts: correct docstring

* signer: address review concerns

* rpc: go fmt -s

* signer: review concerns+ baptize Clef

* signer,node: move Start-functions to separate file

* signer: formatting

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sol linguist-language=Solidity

.github/CODEOWNERS

@@ -9,3 +9,4 @@ les/               @zsfelfoldi
 light/             @zsfelfoldi
 mobile/            @karalabe
 p2p/               @fjl @zsfelfoldi
+whisper/           @gballet @gluk256

.github/no-response.yml

@@ -0,0 +1,11 @@
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 30
+# Label requiring a response
+responseRequiredLabel: more-information-needed
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.gitignore

@@ -42,3 +42,6 @@ profile.cov
 /dashboard/assets/node_modules
 /dashboard/assets/stats.json
 /dashboard/assets/bundle.js
+/dashboard/assets/package-lock.json
+
+**/yarn-error.log

.travis.yml

@@ -6,40 +6,40 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.8.x
+      go: 1.9.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.9.x
+      go: 1.10.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.9.x
+      go: 1.10.x
       script:
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - brew update
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.9.x
+      go: 1.10.x
       env:
         - lint
       git:
@@ -47,14 +47,12 @@ matrix:
       script:
         - go run build/ci.go lint
 
-    # This builder does the Ubuntu PPA and Linux Azure uploads
+    # This builder does the Ubuntu PPA upload
     - os: linux
       dist: trusty
-      sudo: required
-      go: 1.9.x
+      go: 1.10.x
       env:
         - ubuntu-ppa
-        - azure-linux
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -63,11 +61,25 @@ matrix:
             - devscripts
             - debhelper
             - dput
-            - gcc-multilib
             - fakeroot
       script:
-        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+
+    # This builder does the Linux Azure uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.10.x
+      env:
+        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+      script:
+        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
         - go run build/ci.go install -arch 386
@@ -91,7 +103,7 @@ matrix:
       dist: trusty
       services:
         - docker
-      go: 1.9.x
+      go: 1.10.x
       env:
         - azure-linux-mips
       git:
@@ -135,7 +147,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.10.1.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -152,7 +164,7 @@ matrix:
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: 1.9.x
+      go: 1.10.x
       env:
         - azure-osx
         - azure-ios
@@ -181,8 +193,7 @@ matrix:
     # This builder does the Azure archive purges to avoid accumulating junk
     - os: linux
       dist: trusty
-      sudo: required
-      go: 1.9.x
+      go: 1.10.x
       env:
         - azure-purge
       git:

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.9-alpine as builder
+FROM golang:1.10-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 
@@ -12,5 +12,11 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.9-alpine as builder
+FROM golang:1.10-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 
@@ -12,4 +12,10 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp

README.md

@@ -142,7 +142,7 @@ Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containe
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
-network via your own programs and not manually through the console. To aid this, Geth has built in
+network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
 exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).

VERSION

@@ -1 +1 @@
-1.8.2
+1.8.4

accounts/abi/abi_test.go

@@ -621,14 +621,16 @@ func TestBareEvents(t *testing.T) {
 // TestUnpackEvent is based on this contract:
 //    contract T {
 //      event received(address sender, uint amount, bytes memo);
+//      event receivedAddr(address sender);
 //      function receive(bytes memo) external payable {
 //        received(msg.sender, msg.value, memo);
+//        receivedAddr(msg.sender);
 //      }
 //    }
 // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt:
 //   receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
 func TestUnpackEvent(t *testing.T) {
-	const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"}]`
+	const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]`
 	abi, err := JSON(strings.NewReader(abiJSON))
 	if err != nil {
 		t.Fatal(err)
@@ -656,6 +658,17 @@ func TestUnpackEvent(t *testing.T) {
 	} else {
 		t.Logf("len(data): %d; received event: %+v", len(data), ev)
 	}
+
+	type ReceivedAddrEvent struct {
+		Address common.Address
+	}
+	var receivedAddrEv ReceivedAddrEvent
+	err = abi.Unpack(&receivedAddrEv, "receivedAddr", data)
+	if err != nil {
+		t.Error(err)
+	} else {
+		t.Logf("len(data): %d; received event: %+v", len(data), receivedAddrEv)
+	}
 }
 
 func TestABI_MethodById(t *testing.T) {

accounts/abi/argument.go

@@ -113,16 +113,8 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa
 	}
 	// If the output interface is a struct, make sure names don't collide
 	if kind == reflect.Struct {
-		exists := make(map[string]bool)
-		for _, arg := range arguments {
-			field := capitalise(arg.Name)
-			if field == "" {
-				return fmt.Errorf("abi: purely underscored output cannot unpack to struct")
-			}
-			if exists[field] {
-				return fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", field)
-			}
-			exists[field] = true
+		if err := requireUniqueStructFieldNames(arguments); err != nil {
+			return err
 		}
 	}
 	for i, arg := range arguments.NonIndexed() {
@@ -131,14 +123,9 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa
 
 		switch kind {
 		case reflect.Struct:
-			name := capitalise(arg.Name)
-			for j := 0; j < typ.NumField(); j++ {
-				// TODO read tags: `abi:"fieldName"`
-				if typ.Field(j).Name == name {
-					if err := set(value.Field(j), reflectValue, arg); err != nil {
-						return err
-					}
-				}
+			err := unpackStruct(value, reflectValue, arg)
+			if err != nil {
+				return err
 			}
 		case reflect.Slice, reflect.Array:
 			if value.Len() < i {
@@ -165,8 +152,20 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf
 		return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues))
 	}
 	elem := reflect.ValueOf(v).Elem()
+	kind := elem.Kind()
 	reflectValue := reflect.ValueOf(marshalledValues[0])
+
+	if kind == reflect.Struct {
+		//make sure names don't collide
+		if err := requireUniqueStructFieldNames(arguments); err != nil {
+			return err
+		}
+
+		return unpackStruct(elem, reflectValue, arguments[0])
+	}
+
 	return set(elem, reflectValue, arguments.NonIndexed()[0])
+
 }
 
 // Computes the full size of an array;
@@ -278,3 +277,18 @@ func capitalise(input string) string {
 	}
 	return strings.ToUpper(input[:1]) + input[1:]
 }
+
+//unpackStruct extracts each argument into its corresponding struct field
+func unpackStruct(value, reflectValue reflect.Value, arg Argument) error {
+	name := capitalise(arg.Name)
+	typ := value.Type()
+	for j := 0; j < typ.NumField(); j++ {
+		// TODO read tags: `abi:"fieldName"`
+		if typ.Field(j).Name == name {
+			if err := set(value.Field(j), reflectValue, arg); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

accounts/abi/bind/backends/simulated.go

@@ -307,9 +307,9 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 
 	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
-			block.AddTx(tx)
+			block.AddTxWithChain(b.blockchain, tx)
 		}
-		block.AddTx(tx)
+		block.AddTxWithChain(b.blockchain, tx)
 	})
 	statedb, _ := b.blockchain.State()
 

accounts/abi/bind/bind.go

@@ -385,8 +385,7 @@ var methodNormalizer = map[Lang]func(string) string{
 	LangJava: decapitalise,
 }
 
-// capitalise makes the first character of a string upper case, also removing any
-// prefixing underscores from the variable names.
+// capitalise makes a camel-case string which starts with an upper case character.
 func capitalise(input string) string {
 	for len(input) > 0 && input[0] == '_' {
 		input = input[1:]
@@ -394,12 +393,42 @@ func capitalise(input string) string {
 	if len(input) == 0 {
 		return ""
 	}
-	return strings.ToUpper(input[:1]) + input[1:]
+	return toCamelCase(strings.ToUpper(input[:1]) + input[1:])
 }
 
-// decapitalise makes the first character of a string lower case.
+// decapitalise makes a camel-case string which starts with a lower case character.
 func decapitalise(input string) string {
-	return strings.ToLower(input[:1]) + input[1:]
+	for len(input) > 0 && input[0] == '_' {
+		input = input[1:]
+	}
+	if len(input) == 0 {
+		return ""
+	}
+	return toCamelCase(strings.ToLower(input[:1]) + input[1:])
+}
+
+// toCamelCase converts an under-score string to a camel-case string
+func toCamelCase(input string) string {
+	toupper := false
+
+	result := ""
+	for k, v := range input {
+		switch {
+		case k == 0:
+			result = strings.ToUpper(string(input[0]))
+
+		case toupper:
+			result += strings.ToUpper(string(v))
+			toupper = false
+
+		case v == '_':
+			toupper = true
+
+		default:
+			result += string(v)
+		}
+	}
+	return result
 }
 
 // structured checks whether a list of ABI data types has enough information to

accounts/abi/bind/bind_test.go

@@ -425,7 +425,7 @@ var bindTests = []struct {
 			}
 		`,
 	},
-	// Tests that gas estimation works for contracts with	weird gas mechanics too.
+	// Tests that gas estimation works for contracts with weird gas mechanics too.
 	{
 		`FunkyGasPattern`,
 		`
@@ -506,6 +506,7 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Tests that methods and returns with underscores inside work correctly.
 	{
 		`Underscorer`,
 		`
@@ -518,7 +519,7 @@ var bindTests = []struct {
 			}
 			function LowerUpperCollision() constant returns (int _res, int Res) {
 				return (1, 2);
-		  }
+			}
 			function UpperLowerCollision() constant returns (int _Res, int res) {
 				return (1, 2);
 			}
@@ -531,9 +532,12 @@ var bindTests = []struct {
 			function AllPurelyUnderscoredOutput() constant returns (int _, int __) {
 				return (1, 2);
 			}
+			function _under_scored_func() constant returns (int _int) {
+				return 0;
+			}
 		}
-		`, `6060604052341561000f57600080fd5b6103498061001e6000396000f300606060405260043610610083576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806303a592131461008857806367e6633d146100b85780639df484851461014d578063af7486ab1461017d578063b564b34d146101ad578063e02ab24d146101dd578063e409ca451461020d575b600080fd5b341561009357600080fd5b61009b61023d565b604051808381526020018281526020019250505060405180910390f35b34156100c357600080fd5b6100cb610252565b6040518083815260200180602001828103825283818151815260200191508051906020019080838360005b838110156101115780820151818401526020810190506100f6565b50505050905090810190601f16801561013e5780820380516001836020036101000a031916815260200191505b50935050505060405180910390f35b341561015857600080fd5b6101606102a0565b604051808381526020018281526020019250505060405180910390f35b341561018857600080fd5b6101906102b5565b604051808381526020018281526020019250505060405180910390f35b34156101b857600080fd5b6101c06102ca565b604051808381526020018281526020019250505060405180910390f35b34156101e857600080fd5b6101f06102df565b604051808381526020018281526020019250505060405180910390f35b341561021857600080fd5b6102206102f4565b604051808381526020018281526020019250505060405180910390f35b60008060016002819150809050915091509091565b600061025c610309565b61013a8090506040805190810160405280600281526020017f7069000000000000000000000000000000000000000000000000000000000000815250915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b6020604051908101604052806000815250905600a165627a7a72305820c11dcfa136fc7d182ee4d34f0b12d988496228f7e2d02d2b5376d996ca1743d00029`,
-		`[{"constant":true,"inputs":[],"name":"LowerUpperCollision","outputs":[{"name":"_res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UnderscoredOutput","outputs":[{"name":"_int","type":"int256"},{"name":"_string","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"PurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperLowerCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"AllPurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"__","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperUpperCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"LowerLowerCollision","outputs":[{"name":"_res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"}]`,
+		`, `6060604052341561000f57600080fd5b6103858061001e6000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806303a592131461009357806346546dbe146100c357806367e6633d146100ec5780639df4848514610181578063af7486ab146101b1578063b564b34d146101e1578063e02ab24d14610211578063e409ca4514610241575b600080fd5b341561009e57600080fd5b6100a6610271565b604051808381526020018281526020019250505060405180910390f35b34156100ce57600080fd5b6100d6610286565b6040518082815260200191505060405180910390f35b34156100f757600080fd5b6100ff61028e565b6040518083815260200180602001828103825283818151815260200191508051906020019080838360005b8381101561014557808201518184015260208101905061012a565b50505050905090810190601f1680156101725780820380516001836020036101000a031916815260200191505b50935050505060405180910390f35b341561018c57600080fd5b6101946102dc565b604051808381526020018281526020019250505060405180910390f35b34156101bc57600080fd5b6101c46102f1565b604051808381526020018281526020019250505060405180910390f35b34156101ec57600080fd5b6101f4610306565b604051808381526020018281526020019250505060405180910390f35b341561021c57600080fd5b61022461031b565b604051808381526020018281526020019250505060405180910390f35b341561024c57600080fd5b610254610330565b604051808381526020018281526020019250505060405180910390f35b60008060016002819150809050915091509091565b600080905090565b6000610298610345565b61013a8090506040805190810160405280600281526020017f7069000000000000000000000000000000000000000000000000000000000000815250915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b6020604051908101604052806000815250905600a165627a7a72305820d1a53d9de9d1e3d55cb3dc591900b63c4f1ded79114f7b79b332684840e186a40029`,
+		`[{"constant":true,"inputs":[],"name":"LowerUpperCollision","outputs":[{"name":"_res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"_under_scored_func","outputs":[{"name":"_int","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UnderscoredOutput","outputs":[{"name":"_int","type":"int256"},{"name":"_string","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"PurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperLowerCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"AllPurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"__","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperUpperCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"LowerLowerCollision","outputs":[{"name":"_res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"}]`,
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
@@ -562,6 +566,7 @@ var bindTests = []struct {
 			a, b, _ = underscorer.UpperUpperCollision(nil)
 			a, b, _ = underscorer.PurelyUnderscoredOutput(nil)
 			a, b, _ = underscorer.AllPurelyUnderscoredOutput(nil)
+			a, _ = underscorer.UnderScoredFunc(nil)
 
 			fmt.Println(a, b, err)
 		`,
@@ -801,7 +806,8 @@ var bindTests = []struct {
 			// (See accounts/abi/unpack_test.go for more extensive testing)
 			if retrievedArr[4][3][2] != testArr[4][3][2] {
 				t.Fatalf("Retrieved value does not match expected value! got: %d, expected: %d. %v", retrievedArr[4][3][2], testArr[4][3][2], err)
-			}`,
+			}
+		`,
 	},
 }
 

accounts/abi/numbers.go

@@ -25,23 +25,23 @@ import (
 )
 
 var (
-	big_t      = reflect.TypeOf(&big.Int{})
-	derefbig_t = reflect.TypeOf(big.Int{})
-	uint8_t    = reflect.TypeOf(uint8(0))
-	uint16_t   = reflect.TypeOf(uint16(0))
-	uint32_t   = reflect.TypeOf(uint32(0))
-	uint64_t   = reflect.TypeOf(uint64(0))
-	int_t      = reflect.TypeOf(int(0))
-	int8_t     = reflect.TypeOf(int8(0))
-	int16_t    = reflect.TypeOf(int16(0))
-	int32_t    = reflect.TypeOf(int32(0))
-	int64_t    = reflect.TypeOf(int64(0))
-	address_t  = reflect.TypeOf(common.Address{})
-	int_ts     = reflect.TypeOf([]int(nil))
-	int8_ts    = reflect.TypeOf([]int8(nil))
-	int16_ts   = reflect.TypeOf([]int16(nil))
-	int32_ts   = reflect.TypeOf([]int32(nil))
-	int64_ts   = reflect.TypeOf([]int64(nil))
+	bigT      = reflect.TypeOf(&big.Int{})
+	derefbigT = reflect.TypeOf(big.Int{})
+	uint8T    = reflect.TypeOf(uint8(0))
+	uint16T   = reflect.TypeOf(uint16(0))
+	uint32T   = reflect.TypeOf(uint32(0))
+	uint64T   = reflect.TypeOf(uint64(0))
+	intT      = reflect.TypeOf(int(0))
+	int8T     = reflect.TypeOf(int8(0))
+	int16T    = reflect.TypeOf(int16(0))
+	int32T    = reflect.TypeOf(int32(0))
+	int64T    = reflect.TypeOf(int64(0))
+	addressT  = reflect.TypeOf(common.Address{})
+	intTS     = reflect.TypeOf([]int(nil))
+	int8TS    = reflect.TypeOf([]int8(nil))
+	int16TS   = reflect.TypeOf([]int16(nil))
+	int32TS   = reflect.TypeOf([]int32(nil))
+	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
@@ -52,7 +52,7 @@ func U256(n *big.Int) []byte {
 // checks whether the given reflect value is signed. This also works for slices with a number type
 func isSigned(v reflect.Value) bool {
 	switch v.Type() {
-	case int_ts, int8_ts, int16_ts, int32_ts, int64_ts, int_t, int8_t, int16_t, int32_t, int64_t:
+	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
 		return true
 	}
 	return false

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbigT {
 		return indirect(v.Elem())
 	}
 	return v
@@ -36,26 +36,26 @@ func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type)
 	switch size {
 	case 8:
 		if unsigned {
-			return reflect.Uint8, uint8_t
+			return reflect.Uint8, uint8T
 		}
-		return reflect.Int8, int8_t
+		return reflect.Int8, int8T
 	case 16:
 		if unsigned {
-			return reflect.Uint16, uint16_t
+			return reflect.Uint16, uint16T
 		}
-		return reflect.Int16, int16_t
+		return reflect.Int16, int16T
 	case 32:
 		if unsigned {
-			return reflect.Uint32, uint32_t
+			return reflect.Uint32, uint32T
 		}
-		return reflect.Int32, int32_t
+		return reflect.Int32, int32T
 	case 64:
 		if unsigned {
-			return reflect.Uint64, uint64_t
+			return reflect.Uint64, uint64T
 		}
-		return reflect.Int64, int64_t
+		return reflect.Int64, int64T
 	}
-	return reflect.Ptr, big_t
+	return reflect.Ptr, bigT
 }
 
 // mustArrayToBytesSlice creates a new byte slice with the exact same size as value
@@ -110,3 +110,19 @@ func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind,
 	}
 	return nil
 }
+
+// requireUniqueStructFieldNames makes sure field names don't collide
+func requireUniqueStructFieldNames(args Arguments) error {
+	exists := make(map[string]bool)
+	for _, arg := range args {
+		field := capitalise(arg.Name)
+		if field == "" {
+			return fmt.Errorf("abi: purely underscored output cannot unpack to struct")
+		}
+		if exists[field] {
+			return fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", field)
+		}
+		exists[field] = true
+	}
+	return nil
+}

accounts/abi/type.go

@@ -135,7 +135,7 @@ func NewType(t string) (typ Type, err error) {
 		typ.Type = reflect.TypeOf(bool(false))
 	case "address":
 		typ.Kind = reflect.Array
-		typ.Type = address_t
+		typ.Type = addressT
 		typ.Size = 20
 		typ.T = AddressTy
 	case "string":

accounts/abi/type_test.go

@@ -46,36 +46,36 @@ func TestTypeRegexp(t *testing.T) {
 		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
 		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
 		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
 		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
 		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
 		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
@@ -84,9 +84,9 @@ func TestTypeRegexp(t *testing.T) {
 		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
 		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
 		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -252,6 +252,9 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes20", common.Address{}, ""},
 		{"address", [20]byte{}, ""},
 		{"address", common.Address{}, ""},
+		{"bytes32[]]", "", "invalid arg type in abi"},
+		{"invalidType", "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil && len(test.err) == 0 {

accounts/abi/unpack_test.go

@@ -56,6 +56,23 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: true,
 	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000000",
+		want: false,
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000001000000000001",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000003",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
 	{
 		def:  `[{"type": "uint32"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",

accounts/url.go

@@ -74,6 +74,22 @@ func (u URL) MarshalJSON() ([]byte, error) {
 	return json.Marshal(u.String())
 }
 
+// UnmarshalJSON parses url.
+func (u *URL) UnmarshalJSON(input []byte) error {
+	var textUrl string
+	err := json.Unmarshal(input, &textUrl)
+	if err != nil {
+		return err
+	}
+	url, err := parseURL(textUrl)
+	if err != nil {
+		return err
+	}
+	u.Scheme = url.Scheme
+	u.Path = url.Path
+	return nil
+}
+
 // Cmp compares x and y and returns:
 //
 //   -1 if x <  y

accounts/usbwallet/hub.go

@@ -127,7 +127,7 @@ func (hub *Hub) refreshWallets() {
 		// breaking the Ledger protocol if that is waiting for user confirmation. This
 		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
 		// need to prevent concurrent comms ourselves. The more elegant solution would
-		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// be to ditch enumeration in favor of hotplug events, but that don't work yet
 		// on Windows so if we need to hack it anyway, this is more elegant for now.
 		hub.commsLock.Lock()
 		if hub.commsPend > 0 { // A confirmation is pending, don't refresh

accounts/usbwallet/wallet.go

@@ -99,7 +99,7 @@ type wallet struct {
 	//
 	// As such, a hardware wallet needs two locks to function correctly. A state
 	// lock can be used to protect the wallet's software-side internal state, which
-	// must not be held exlusively during hardware communication. A communication
+	// must not be held exclusively during hardware communication. A communication
 	// lock can be used to achieve exclusive access to the device itself, this one
 	// however should allow "skipping" waiting for operations that might want to
 	// use the device, but can live without too (e.g. account self-derivation).

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.9.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.9.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.10.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

bmt/bmt.go

@@ -75,7 +75,7 @@ type Hasher struct {
 	blocksize   int         // segment size (size of hash) also for hash.Hash
 	count       int         // segment count
 	size        int         // for hash.Hash same as hashsize
-	cur         int         // cursor position for righmost currently open chunk
+	cur         int         // cursor position for rightmost currently open chunk
 	segment     []byte      // the rightmost open segment (not complete)
 	depth       int         // index of last level
 	result      chan []byte // result channel
@@ -149,7 +149,7 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
 	}
 }
 
-// Drain drains the pool uptil it has no more than n resources
+// Drain drains the pool until it has no more than n resources
 func (self *TreePool) Drain(n int) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
@@ -412,11 +412,10 @@ func (self *Hasher) Reset() {
 
 // ResetWithLength needs to be called before writing to the hasher
 // the argument is supposed to be the byte slice binary representation of
-// the legth of the data subsumed under the hash
+// the length of the data subsumed under the hash
 func (self *Hasher) ResetWithLength(l []byte) {
 	self.Reset()
 	self.blockLength = l
-
 }
 
 // Release gives back the Tree to the pool whereby it unlocks
@@ -531,7 +530,7 @@ func (self *Hasher) finalise(n *Node, i int) (d int) {
 	for {
 		// when the final segment's path is going via left segments
 		// the incoming data is pushed to the parent upon pulling the left
-		// we do not need toogle the state since this condition is
+		// we do not need toggle the state since this condition is
 		// detectable
 		n.unbalanced = isLeft
 		n.right = nil

build/ci-notes.md

@@ -2,12 +2,7 @@
 
 Tagged releases and develop branch commits are available as installable Debian packages
 for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
-Canonical:
-
-- Trusty Tahr (14.04 LTS)
-- Xenial Xerus (16.04 LTS)
-- Yakkety Yak (16.10)
-- Zesty Zapus (17.04)
+Canonical.
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,18 +16,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.9, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.10, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.9 and Debian packaging tools:
+Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.9 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
 
 Create the source packages:
 

build/ci.go

@@ -182,13 +182,13 @@ func doInstall(cmdline []string) {
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
 	if !strings.Contains(runtime.Version(), "devel") {
-		// Figure out the minor version number since we can't textually compare (1.10 < 1.8)
+		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
 		var minor int
 		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
 
-		if minor < 8 {
+		if minor < 9 {
 			log.Println("You have Go version", runtime.Version())
-			log.Println("go-ethereum requires at least Go version 1.8 and cannot")
+			log.Println("go-ethereum requires at least Go version 1.9 and cannot")
 			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
 			os.Exit(1)
 		}
@@ -262,16 +262,6 @@ func goTool(subcmd string, args ...string) *exec.Cmd {
 
 func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
 	cmd := build.GoTool(subcmd, args...)
-	if subcmd == "build" || subcmd == "install" || subcmd == "test" {
-		// Go CGO has a Windows linker error prior to 1.8 (https://github.com/golang/go/issues/8756).
-		// Work around issue by allowing multiple definitions for <1.8 builds.
-		var minor int
-		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
-
-		if runtime.GOOS == "windows" && minor < 8 {
-			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
-		}
-	}
 	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
 	if arch == "" || arch == runtime.GOARCH {
 		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
@@ -736,7 +726,7 @@ func doAndroidArchive(cmdline []string) {
 		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
 	}
 	// Build the Android archive and Maven resources
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
+	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
@@ -776,7 +766,7 @@ func doAndroidArchive(cmdline []string) {
 		if meta.Develop {
 			repo = *deploy + "/content/repositories/snapshots"
 		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
+		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
 			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
 			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
 	}
@@ -787,9 +777,10 @@ func gomobileTool(subcmd string, args ...string) *exec.Cmd {
 	cmd.Args = append(cmd.Args, args...)
 	cmd.Env = []string{
 		"GOPATH=" + build.GOPATH(),
+		"PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"),
 	}
 	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOPATH=") {
+		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") {
 			continue
 		}
 		cmd.Env = append(cmd.Env, e)
@@ -856,7 +847,7 @@ func doXCodeFramework(cmdline []string) {
 	env := build.Env()
 
 	// Build the iOS XCode framework
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
+	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("init"))
 	bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
 

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.9
+Build-Depends: debhelper (>= 8.0.0), golang-1.10
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.9/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

cmd/clef/README.md

@@ -0,0 +1,864 @@
+Clef
+----
+Clef can be used to sign transactions and data and is meant as a replacement for geth's account management.
+This allows DApps not to depend on geth's account management. When a DApp wants to sign data it can send the data to
+the signer, the signer will then provide the user with context and asks the user for permission to sign the data. If
+the users grants the signing request the signer will send the signature back to the DApp.
+  
+This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can
+help in situations when a DApp is connected to a remote node because a local Ethereum node is not available, not
+synchronised with the chain or a particular Ethereum node that has no built-in (or limited) account management.
+  
+Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
+or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
+
+
+## Command line flags
+Clef accepts the following command line options:
+```
+COMMANDS:
+   init    Initialize the signer, generate secret storage
+   attest  Attest that a js-file is to be used
+   addpw   Store a credential for a keystore file
+   help    Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --loglevel value        log level to emit to the screen (default: 4)
+   --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
+   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
+   --nousb                 Disables monitoring for and managing USB hardware wallets
+   --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcport value         HTTP-RPC server listening port (default: 8550)
+   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
+   --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
+   --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
+   --rules value           Enable rule-engine (default: "rules.json")
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --help, -h              show help
+   --version, -v           print the version
+
+```
+
+
+Example:
+```
+signer -keystore /my/keystore -chainid 4
+```
+
+Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works.
+
+## Security model
+
+The security model of the signer is as follows:
+
+* One critical component (the signer binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
+* The signer binary has a well-defined 'external' API.
+* The 'external' API is considered UNTRUSTED.
+* The signer binary also communicates with whatever process that invoked the binary, via stdin/stdout.
+  * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
+
+The general flow for signing a transaction using e.g. geth is as follows:
+![image](sign_flow.png)
+
+In this case, `geth` would be started with `--externalsigner=http://localhost:8550` and would relay requests to `eth.sendTransaction`.
+
+## TODOs
+
+Some snags and todos
+
+* [ ] The signer should take a startup param "--no-change", for UIs that do not contain the capability
+   to perform changes to things, only approve/deny. Such a UI should be able to start the signer in
+   a more secure mode by telling it that it only wants approve/deny capabilities.
+
+* [x] It would be nice if the signer could collect new 4byte-id:s/method selectors, and have a
+secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for
+inclusion upstream.
+
+* It should be possible to configure the signer to check if an account is indeed known to it, before
+passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate
+accounts if it immediately returned "unknown account".
+* [x] It should be possible to configure the signer to auto-allow listing (certain) accounts, instead of asking every time.
+* [x] Done Upon startup, the signer should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode),
+invoking methods with the following info:
+  * [x] Version info about the signer
+  * [x] Address of API (http/ipc)
+  * [ ] List of known accounts
+* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] `account_signRawTransaction`
+* [ ] `account_bulkSignTransactions([] transactions)` should
+   * only exist if enabled via config/flag
+   * only allow non-data-sending transactions
+   * all txs must use the same `from`-account
+   * let the user confirm, showing
+      * the total amount
+      * the number of unique recipients
+
+* Geth todos
+    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is
+put together is a bit of a hack into the http server. This could probably be greatly improved
+    - Relay: Geth should be started in `geth --external_signer localhost:8550`.
+    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This
+  type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which
+  retains the original input.
+    - The Geth api should switch to use the same type, and relay `to`-account verbatim to the external api.
+
+* [x] Storage
+    * [x] An encrypted key-value storage should be implemented
+    * See [rules.md](rules.md) for more info about this.
+
+* Another potential thing to introduce is pairing.
+  * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
+  * Thus geth/mist/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
+  * This feature would make the addition of rules less dangerous.
+
+* Wallets / accounts. Add API methods for wallets.
+
+## Communication
+
+### External API
+
+The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRPC standard as Geth. The messages are
+expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
+
+Some of these call can require user interaction. Clients must be aware that responses
+may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+
+The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
+that requests have any authority.
+
+### UI API
+
+The signer has one native console-based UI, for operation without any standalone tools.
+However, there is also an API to communicate with an external UI. To enable that UI,
+the signer needs to be executed with the `--stdio-ui` option, which allocates the
+`stdin`/`stdout` for the UI-api.
+
+An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
+
+The model is as follows:
+
+* The user starts the UI app (`pythonsigner.py`).
+* The UI app starts the `signer` with `--stdio-ui`, and listens to the
+process output for confirmation-requests.
+* The `signer` opens the external http api.
+* When the `signer` receives requests, it sends a `jsonrpc` request via `stdout`.
+* The UI app prompts the user accordingly, and responds to the `signer`
+* The `signer` signs (or not), and responds to the original request.
+
+## External API
+
+See the [external api changelog](extapi_changelog.md) for information about changes to this API.
+
+### Encoding
+- number: positive integers that are hex encoded
+- data: hex encoded data
+- string: ASCII string
+
+All hex encoded values must be prefixed with `0x`.
+
+## Methods
+
+### account_new
+
+#### Create new password protected account
+
+The signer will generate a new private key, encrypts it according to [web3 keystore spec](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) and stores it in the keystore directory.
+The client is responsible for creating a backup of the keystore. If the keystore is lost there is no method of retrieving lost accounts.
+
+#### Arguments
+
+None
+
+#### Result
+  - address [string]: account address that is derived from the generated key
+  - url [string]: location of the keyfile
+  
+#### Sample call
+```json
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "method": "account_new",
+  "params": []
+}
+
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+    "url": "keystore:///my/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+  }
+}
+```
+
+### account_list
+
+#### List available accounts
+   List all accounts that this signer currently manages
+
+#### Arguments
+
+None
+
+#### Result
+  - array with account records:
+     - account.address [string]: account address that is derived from the generated key
+     - account.type [string]: type of the 
+     - account.url [string]: location of the account
+  
+#### Sample call
+```json
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "method": "account_list"
+}
+
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "result": [
+    {
+      "address": "0xafb2f771f58513609765698f65d3f2f0224a956f",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T07-26-47.162109726Z--afb2f771f58513609765698f65d3f2f0224a956f"
+    },
+    {
+      "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+    }
+  ]
+}
+```
+
+### account_signTransaction
+
+#### Sign transactions
+   Signs a transactions and responds with the signed transaction in RLP encoded form.
+
+#### Arguments
+  2. transaction object:
+     - `from` [address]: account to send the transaction from
+     - `to` [address]: receiver account. If omitted or `0x`, will cause contract creation.
+     - `gas` [number]: maximum amount of gas to burn
+     - `gasPrice` [number]: gas price
+     - `value` [number:optional]: amount of Wei to send with the transaction
+     - `data` [data:optional]:  input data
+     - `nonce` [number]: account nonce
+  3. method signature [string:optional]
+     - The method signature, if present, is to aid decoding the calldata. Should consist of `methodname(paramtype,...)`, e.g. `transfer(uint256,address)`. The signer may use this data to parse the supplied calldata, and show the user. The data, however, is considered totally untrusted, and reliability is not expected.
+
+
+#### Result
+  - signed transaction in RLP encoded form [data]
+  
+#### Sample call
+```json
+{
+  "id": 2,
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+      "gas": "0x55555",
+      "gasPrice": "0x1234",
+      "input": "0xabcd",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x1234"
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "error": {
+    "code": -32000,
+    "message": "Request denied"
+  }
+}
+```
+#### Sample call with ABI-data
+
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+      "gas": "0x333",
+      "gasPrice": "0x1",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
+    },
+    "safeSend(address)"
+  ],
+  "id": 67
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "result": {
+    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+    "tx": {
+      "nonce": "0x0",
+      "gasPrice": "0x1",
+      "gas": "0x333",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+      "v": "0x26",
+      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
+      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
+    }
+  }
+}
+```
+
+Bash example:
+```bash
+#curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+
+{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
+```
+
+
+### account_sign
+
+#### Sign data
+   Signs a chunk of data and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [data]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "method": "account_sign",
+  "params": [
+    "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+    "0xaabbccdd"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "result": "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+}
+```
+
+### account_ecRecover
+
+#### Recover address
+   Derive the address from the account that was used to sign data from the data and signature.
+   
+#### Arguments
+  - data [data]: data that was signed
+  - signature [data]: the signature to verify
+
+#### Result
+  - derived account [address]
+  
+#### Sample call
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "method": "account_ecRecover",
+  "params": [
+    "0xaabbccdd",
+    "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
+}
+
+```
+
+### account_import
+
+#### Import account
+   Import a private key into the keystore. The imported key is expected to be encrypted according to the web3 keystore
+   format.
+   
+#### Arguments
+  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export) 
+
+#### Result
+  - imported key [object]:
+     - key.address [address]: address of the imported key
+     - key.type [string]: type of the account
+     - key.url [string]: key URL
+  
+#### Sample call
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "method": "account_import",
+  "params": [
+    {
+      "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+      "crypto": {
+        "cipher": "aes-128-ctr",
+        "cipherparams": {
+          "iv": "401c39a7c7af0388491c3d3ecb39f532"
+        },
+        "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+        "kdf": "scrypt",
+        "kdfparams": {
+          "dklen": 32,
+          "n": 262144,
+          "p": 1,
+          "r": 8,
+          "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+        },
+        "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+      },
+      "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+      "version": 3
+    },
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xc7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "type": "account",
+    "url": "keystore:///tmp/keystore/UTC--2017-08-24T11-00-42.032024108Z--c7412fc59930fd90099c917a50e5f11d0934b2f5"
+  }
+}
+```
+
+### account_export
+
+#### Export account from keystore
+   Export a private key from the keystore. The exported private key is encrypted with the original passphrase. When the
+   key is imported later this passphrase is required.
+   
+#### Arguments
+  - account [address]: export private key that is associated with this account
+
+#### Result
+  - exported key, see [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) for
+  more information
+  
+#### Sample call
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "method": "account_export",
+  "params": [
+    "0xc7412fc59930fd90099c917a50e5f11d0934b2f5"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "crypto": {
+      "cipher": "aes-128-ctr",
+      "cipherparams": {
+        "iv": "401c39a7c7af0388491c3d3ecb39f532"
+      },
+      "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+      "kdf": "scrypt",
+      "kdfparams": {
+        "dklen": 32,
+        "n": 262144,
+        "p": 1,
+        "r": 8,
+        "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+      },
+      "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+    },
+    "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+    "version": 3
+  }
+}
+```
+
+
+
+## UI API
+
+These methods needs to be implemented by a UI listener.
+
+By starting the signer with the switch `--stdio-ui-test`, the signer will invoke all known methods, and expect the UI to respond with
+denials. This can be used during development to ensure that the API is (at least somewhat) correctly implemented.
+See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to perform the 'denial-handshake-test'.
+
+All methods in this API uses object-based parameters, so that there can be no mixups of parameters: each piece of data is accessed by key.
+
+See the [ui api changelog](intapi_changelog.md) for information about changes to this API.
+
+OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
+Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
+things simpler for both parties.
+
+### ApproveTx
+
+Invoked when there's a transaction for approval.
+
+
+#### Sample call
+
+Here's a method invocation:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "Info",
+            "message": "safeSend(address: 0x0000000000000000000000000000000000000012)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48486",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+```
+
+The same method invocation, but with invalid data:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000002000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000002000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "WARNING",
+            "message": "Transaction data did not match ABI-interface: WARNING: Supplied data is stuffed with extra data. \nWant 0000000000000002000000000000000000000000000000000000000000000012\nHave 0000000000000000000000000000000000000000000000000000000000000012\nfor method safeSend(address)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48492",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+
+```
+
+One which has missing `to`, but with no `data`:
+
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 3,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "",
+        "to": null,
+        "gas": "0x0",
+        "gasPrice": "0x0",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": null,
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "CRITICAL",
+            "message": "Tx will create contract with empty code!"
+          }
+        ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+```
+
+### ApproveExport
+
+Invoked when a request to export an account has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 7,
+  "method": "ApproveExport",
+  "params": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ApproveListing
+
+Invoked when a request for account listing has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 5,
+  "method": "ApproveListing",
+  "params": [
+    {
+      "accounts": [
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-20T14-44-54.089682944Z--123409812340981234098123409812deadbeef42",
+          "address": "0x123409812340981234098123409812deadbeef42"
+        },
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-23T21-59-03.199240693Z--cafebabedeadbeef34098123409812deadbeef42",
+          "address": "0xcafebabedeadbeef34098123409812deadbeef42"
+        }
+      ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+
+### ApproveSignData
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 4,
+  "method": "ApproveSignData",
+  "params": [
+    {
+      "address": "0x123409812340981234098123409812deadbeef42",
+      "raw_data": "0x01020304",
+      "message": "\u0019Ethereum Signed Message:\n4\u0001\u0002\u0003\u0004",
+      "hash": "0x7e3a4e7a9d1744bc5c675c25e1234ca8ed9162bd17f78b9085e48047c15ac310",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ShowInfo
+
+The UI should show the info to the user. Does not expect response.
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 9,
+  "method": "ShowInfo",
+  "params": [
+    {
+      "text": "Tests completed"
+    }
+  ]
+}
+
+```
+
+### ShowError
+
+The UI should show the info to the user. Does not expect response.
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ShowError",
+  "params": [
+    {
+      "text": "Testing 'ShowError'"
+    }
+  ]
+}
+
+```
+
+### OnApproved
+
+`OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
+
+When implementing rate-limited rules, this callback should be used.
+
+TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
+
+### OnSignerStartup
+
+This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
+in k/v-form.
+
+Example call:
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+
+```
+
+
+### Rules for UI apis
+
+A UI should conform to the following rules.
+
+* A UI MUST NOT load any external resources that were not embedded/part of the UI package.
+  * For example, not load icons, stylesheets from the internet
+  * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
+* A Graphical UI MUST show the blocky-identicon for ethereum addresses.
+* A UI MUST warn display approproate warning if the destination-account is formatted with invalid checksum.
+* A UI MUST NOT open any ports or services
+  * The signer opens the public port
+* A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
+* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
+* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
+  * The signer provides accounts
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+along with the UI.
+
+

cmd/clef/extapi_changelog.md

@@ -0,0 +1,25 @@
+### Changelog for external API
+
+
+
+#### 2.0.0
+
+* Commit `73abaf04b1372fa4c43201fb1b8019fe6b0a6f8d`, move `from` into `transaction` object in `signTransaction`. This
+makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
+
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/intapi_changelog.md

@@ -0,0 +1,86 @@
+### Changelog for internal API (ui-api)
+
+### 2.0.0
+
+* Modify how `call_info` on a transaction is conveyed. New format:
+
+```
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x82A2A876D39022B3019932D30Cd9c97ad5616813",
+        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x123",
+        "value": "0x10",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+        {
+          "type": "WARNING",
+          "message": "Invalid checksum on to-address"
+        },
+        {
+          "type": "WARNING",
+          "message": "Tx contains data, but provided ABI signature could not be matched: Did not match: test (0 matches)"
+        }
+      ],
+      "meta": {
+        "remote": "127.0.0.1:54286",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+```
+
+#### 1.2.0
+
+* Add `OnStartup` method, to provide the UI with information about what API version
+the signer uses (both internal and external) aswell as build-info and external api.
+
+Example call:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+```
+
+#### 1.1.0
+
+* Add `OnApproved` method
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/main.go

@@ -0,0 +1,640 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// signer is a utility that can be used so sign transactions and
+// arbitrary data.
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"encoding/hex"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/rules"
+	"github.com/ethereum/go-ethereum/signer/storage"
+	"gopkg.in/urfave/cli.v1"
+	"os/signal"
+)
+
+// ExternalApiVersion -- see extapi_changelog.md
+const ExternalApiVersion = "2.0.0"
+
+// InternalApiVersion -- see intapi_changelog.md
+const InternalApiVersion = "2.0.0"
+
+const legalWarning = `
+WARNING! 
+
+Clef is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing. 
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE! 
+
+`
+
+var (
+	logLevelFlag = cli.IntFlag{
+		Name:  "loglevel",
+		Value: 4,
+		Usage: "log level to emit to the screen",
+	}
+	keystoreFlag = cli.StringFlag{
+		Name:  "keystore",
+		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
+		Usage: "Directory for the keystore",
+	}
+	configdirFlag = cli.StringFlag{
+		Name:  "configdir",
+		Value: DefaultConfigDir(),
+		Usage: "Directory for Clef configuration",
+	}
+	rpcPortFlag = cli.IntFlag{
+		Name:  "rpcport",
+		Usage: "HTTP-RPC server listening port",
+		Value: node.DefaultHTTPPort + 5,
+	}
+	signerSecretFlag = cli.StringFlag{
+		Name:  "signersecret",
+		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+	}
+	dBFlag = cli.StringFlag{
+		Name:  "4bytedb",
+		Usage: "File containing 4byte-identifiers",
+		Value: "./4byte.json",
+	}
+	customDBFlag = cli.StringFlag{
+		Name:  "4bytedb-custom",
+		Usage: "File used for writing new 4byte-identifiers submitted via API",
+		Value: "./4byte-custom.json",
+	}
+	auditLogFlag = cli.StringFlag{
+		Name:  "auditlog",
+		Usage: "File used to emit audit logs. Set to \"\" to disable",
+		Value: "audit.log",
+	}
+	ruleFlag = cli.StringFlag{
+		Name:  "rules",
+		Usage: "Enable rule-engine",
+		Value: "rules.json",
+	}
+	stdiouiFlag = cli.BoolFlag{
+		Name: "stdio-ui",
+		Usage: "Use STDIN/STDOUT as a channel for an external UI. " +
+			"This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " +
+			"interface, and can be used when Clef is started by an external process.",
+	}
+	testFlag = cli.BoolFlag{
+		Name:  "stdio-ui-test",
+		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
+	}
+	app         = cli.NewApp()
+	initCommand = cli.Command{
+		Action:    utils.MigrateFlags(initializeSecrets),
+		Name:      "init",
+		Usage:     "Initialize the signer, generate secret storage",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+		},
+		Description: `
+The init command generates a master seed which Clef can use to store credentials and data needed for 
+the rule-engine to work.`,
+	}
+	attestCommand = cli.Command{
+		Action:    utils.MigrateFlags(attestFile),
+		Name:      "attest",
+		Usage:     "Attest that a js-file is to be used",
+		ArgsUsage: "<sha256sum>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The attest command stores the sha256 of the rule.js-file that you want to use for automatic processing of 
+incoming requests. 
+
+Whenever you make an edit to the rule file, you need to use attestation to tell 
+Clef that the file is 'safe' to execute.`,
+	}
+
+	addCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(addCredential),
+		Name:      "addpw",
+		Usage:     "Store a credential for a keystore file",
+		ArgsUsage: "<address> <password>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+remove any stored credential for that address (keyfile)
+`,
+	}
+)
+
+func init() {
+	app.Name = "Clef"
+	app.Usage = "Manage Ethereum account operations"
+	app.Flags = []cli.Flag{
+		logLevelFlag,
+		keystoreFlag,
+		configdirFlag,
+		utils.NetworkIdFlag,
+		utils.LightKDFFlag,
+		utils.NoUSBFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCVirtualHostsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+		utils.RPCEnabledFlag,
+		rpcPortFlag,
+		signerSecretFlag,
+		dBFlag,
+		customDBFlag,
+		auditLogFlag,
+		ruleFlag,
+		stdiouiFlag,
+		testFlag,
+	}
+	app.Action = signer
+	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+
+}
+func main() {
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func initializeSecrets(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	configDir := c.String(configdirFlag.Name)
+
+	masterSeed := make([]byte, 256)
+	n, err := io.ReadFull(rand.Reader, masterSeed)
+	if err != nil {
+		return err
+	}
+	if n != len(masterSeed) {
+		return fmt.Errorf("failed to read enough random")
+	}
+	err = os.Mkdir(configDir, 0700)
+	if err != nil && !os.IsExist(err) {
+		return err
+	}
+	location := filepath.Join(configDir, "secrets.dat")
+	if _, err := os.Stat(location); err == nil {
+		return fmt.Errorf("file %v already exists, will not overwrite", location)
+	}
+	err = ioutil.WriteFile(location, masterSeed, 0700)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("A master seed has been generated into %s\n", location)
+	fmt.Printf(`
+This is required to be able to store credentials, such as : 
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it. 
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+
+`)
+	return nil
+}
+func attestFile(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+	// Initialize the encrypted storages
+	configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey)
+	val := ctx.Args().First()
+	configStorage.Put("ruleset_sha256", val)
+	log.Info("Ruleset attestation updated", "sha256", val)
+	return nil
+}
+
+func addCredential(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires at leaste one argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+
+	// Initialize the encrypted storages
+	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	key := ctx.Args().First()
+	value := ""
+	if len(ctx.Args()) > 1 {
+		value = ctx.Args().Get(1)
+	}
+	pwStorage.Put(key, value)
+	log.Info("Credential store updated", "key", key)
+	return nil
+}
+
+func initialize(c *cli.Context) error {
+	// Set up the logger to print everything
+	logOutput := os.Stdout
+	if c.Bool(stdiouiFlag.Name) {
+		logOutput = os.Stderr
+		// If using the stdioui, we can't do the 'confirm'-flow
+		fmt.Fprintf(logOutput, legalWarning)
+	} else {
+		if !confirm(legalWarning) {
+			return fmt.Errorf("aborted by user")
+		}
+	}
+
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(logOutput, log.TerminalFormat(true))))
+	return nil
+}
+
+func signer(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	var (
+		ui core.SignerUI
+	)
+	if c.Bool(stdiouiFlag.Name) {
+		log.Info("Using stdin/stdout as UI-channel")
+		ui = core.NewStdIOUI()
+	} else {
+		log.Info("Using CLI as UI-channel")
+		ui = core.NewCommandlineUI()
+	}
+	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+
+	var (
+		api core.ExternalAPI
+	)
+
+	configDir := c.String(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c); err != nil {
+		log.Info("No master seed provided, rules disabled")
+	} else {
+
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+
+		// Generate domain specific keys
+		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
+		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+		// Initialize the encrypted storages
+		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
+		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+
+		//Do we have a rule-file?
+		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		if err != nil {
+			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
+		} else {
+			hasher := sha256.New()
+			hasher.Write(ruleJS)
+			shasum := hasher.Sum(nil)
+			storedShasum := configStorage.Get("ruleset_sha256")
+			if storedShasum != hex.EncodeToString(shasum) {
+				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+			} else {
+				// Initialize rules
+				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
+				if err != nil {
+					utils.Fatalf(err.Error())
+				}
+				ruleEngine.Init(string(ruleJS))
+				ui = ruleEngine
+				log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
+			}
+		}
+	}
+
+	apiImpl := core.NewSignerAPI(
+		c.Int64(utils.NetworkIdFlag.Name),
+		c.String(keystoreFlag.Name),
+		c.Bool(utils.NoUSBFlag.Name),
+		ui, db,
+		c.Bool(utils.LightKDFFlag.Name))
+
+	api = apiImpl
+
+	// Audit logging
+	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+		api, err = core.NewAuditLogger(logfile, api)
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		log.Info("Audit logs configured", "file", logfile)
+	}
+	// register signer API with server
+	var (
+		extapiUrl = "n/a"
+		ipcApiUrl = "n/a"
+	)
+	rpcApi := []rpc.API{
+		{
+			Namespace: "account",
+			Public:    true,
+			Service:   api,
+			Version:   "1.0"},
+	}
+	if c.Bool(utils.RPCEnabledFlag.Name) {
+
+		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
+		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
+
+		// start http server
+		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts)
+		if err != nil {
+			utils.Fatalf("Could not start RPC api: %v", err)
+		}
+		extapiUrl = fmt.Sprintf("http://%s", httpEndpoint)
+		log.Info("HTTP endpoint opened", "url", extapiUrl)
+
+		defer func() {
+			listener.Close()
+			log.Info("HTTP endpoint closed", "url", httpEndpoint)
+		}()
+
+	}
+	if !c.Bool(utils.IPCDisabledFlag.Name) {
+		if c.IsSet(utils.IPCPathFlag.Name) {
+			ipcApiUrl = c.String(utils.IPCPathFlag.Name)
+		} else {
+			ipcApiUrl = filepath.Join(configDir, "clef.ipc")
+		}
+
+		listener, _, err := rpc.StartIPCEndpoint(func() bool { return true }, ipcApiUrl, rpcApi)
+		if err != nil {
+			utils.Fatalf("Could not start IPC api: %v", err)
+		}
+		log.Info("IPC endpoint opened", "url", ipcApiUrl)
+		defer func() {
+			listener.Close()
+			log.Info("IPC endpoint closed", "url", ipcApiUrl)
+		}()
+
+	}
+
+	if c.Bool(testFlag.Name) {
+		log.Info("Performing UI test")
+		go testExternalUI(apiImpl)
+	}
+	ui.OnSignerStartup(core.StartupInfo{
+		Info: map[string]interface{}{
+			"extapi_version": ExternalApiVersion,
+			"intapi_version": InternalApiVersion,
+			"extapi_http":    extapiUrl,
+			"extapi_ipc":     ipcApiUrl,
+		},
+	})
+
+	abortChan := make(chan os.Signal)
+	signal.Notify(abortChan, os.Interrupt)
+
+	sig := <-abortChan
+	log.Info("Exiting...", "signal", sig)
+
+	return nil
+}
+
+// splitAndTrim splits input separated by a comma
+// and trims excessive white space from the substrings.
+func splitAndTrim(input string) []string {
+	result := strings.Split(input, ",")
+	for i, r := range result {
+		result[i] = strings.TrimSpace(r)
+	}
+	return result
+}
+
+// DefaultConfigDir is the default config directory to use for the vaults and other
+// persistence requirements.
+func DefaultConfigDir() string {
+	// Try to place the data folder in the user's home dir
+	home := homeDir()
+	if home != "" {
+		if runtime.GOOS == "darwin" {
+			return filepath.Join(home, "Library", "Signer")
+		} else if runtime.GOOS == "windows" {
+			return filepath.Join(home, "AppData", "Roaming", "Signer")
+		} else {
+			return filepath.Join(home, ".clef")
+		}
+	}
+	// As we cannot guess a stable location, return empty and handle later
+	return ""
+}
+
+func homeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+	return ""
+}
+func readMasterKey(ctx *cli.Context) ([]byte, error) {
+	var (
+		file      string
+		configDir = ctx.String(configdirFlag.Name)
+	)
+	if ctx.IsSet(signerSecretFlag.Name) {
+		file = ctx.String(signerSecretFlag.Name)
+	} else {
+		file = filepath.Join(configDir, "secrets.dat")
+	}
+	if err := checkFile(file); err != nil {
+		return nil, err
+	}
+	masterKey, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	if len(masterKey) < 256 {
+		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	}
+	// Create vault location
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	err = os.Mkdir(vaultLocation, 0700)
+	if err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	//!TODO, use KDF to stretch the master key
+	//			stretched_key := stretch_key(master_key)
+
+	return masterKey, nil
+}
+
+// checkFile is a convenience function to check if a file
+// * exists
+// * is mode 0600
+func checkFile(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("failed stat on %s: %v", filename, err)
+	}
+	// Check the unix permission bits
+	if info.Mode().Perm()&077 != 0 {
+		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
+	}
+	return nil
+}
+
+// confirm displays a text and asks for user confirmation
+func confirm(text string) bool {
+	fmt.Printf(text)
+	fmt.Printf("\nEnter 'ok' to proceed:\n>")
+
+	text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+
+	if text := strings.TrimSpace(text); text == "ok" {
+		return true
+	}
+	return false
+}
+
+func testExternalUI(api *core.SignerAPI) {
+
+	ctx := context.WithValue(context.Background(), "remote", "clef binary")
+	ctx = context.WithValue(ctx, "scheme", "in-proc")
+	ctx = context.WithValue(ctx, "local", "main")
+
+	errs := make([]string, 0)
+
+	api.UI.ShowInfo("Testing 'ShowInfo'")
+	api.UI.ShowError("Testing 'ShowError'")
+
+	checkErr := func(method string, err error) {
+		if err != nil && err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+		}
+	}
+	var err error
+
+	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
+	checkErr("SignTransaction", err)
+	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+	checkErr("Sign", err)
+	_, err = api.List(ctx)
+	checkErr("List", err)
+	_, err = api.New(ctx)
+	checkErr("New", err)
+	_, err = api.Export(ctx, common.Address{})
+	checkErr("Export", err)
+	_, err = api.Import(ctx, json.RawMessage{})
+	checkErr("Import", err)
+
+	api.UI.ShowInfo("Tests completed")
+
+	if len(errs) > 0 {
+		log.Error("Got errors")
+		for _, e := range errs {
+			log.Error(e)
+		}
+	} else {
+		log.Info("No errors")
+	}
+
+}
+
+/**
+//Create Account
+
+curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_new","params":["test"],"id":67}' localhost:8550
+
+// List accounts
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_list","params":[""],"id":67}' http://localhost:8550/
+
+// Make Transaction
+// safeSend(0x12)
+// 4401a6e40000000000000000000000000000000000000000000000000000000000000012
+
+// supplied abi
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"test"],"id":67}' http://localhost:8550/
+
+// Not supplied
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"}],"id":67}' http://localhost:8550/
+
+// Sign data
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_sign","params":["0x694267f14675d7e1b9494fd8d72fefe1755710fa","bazonk gaz baz"],"id":67}' http://localhost:8550/
+
+
+**/

cmd/clef/pythonsigner.py

@@ -0,0 +1,179 @@
+import os,sys, subprocess
+from tinyrpc.transports import ServerTransport
+from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
+from tinyrpc.dispatch import public,RPCDispatcher
+from tinyrpc.server import RPCServer
+
+""" This is a POC example of how to write a custom UI for Clef. The UI starts the
+clef process with the '--stdio-ui' option, and communicates with clef using standard input / output.
+
+The standard input/output is a relatively secure way to communicate, as it does not require opening any ports
+or IPC files. Needless to say, it does not protect against memory inspection mechanisms where an attacker
+can access process memory."""
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib as urlparse
+
+class StdIOTransport(ServerTransport):
+    """ Uses std input/output for RPC """
+    def receive_message(self):
+        return None, urlparse.unquote(sys.stdin.readline())
+
+    def send_reply(self, context, reply):
+        print(reply)
+
+class PipeTransport(ServerTransport):
+    """ Uses std a pipe for RPC """
+
+    def __init__(self,input, output):
+        self.input = input
+        self.output = output
+
+    def receive_message(self):
+        data = self.input.readline()
+        print(">> {}".format( data))
+        return None, urlparse.unquote(data)
+
+    def send_reply(self, context, reply):
+        print("<< {}".format( reply))
+        self.output.write(reply)
+        self.output.write("\n")
+
+class StdIOHandler():
+
+    def __init__(self):
+        pass
+
+    @public
+    def ApproveTx(self,req):
+        """
+        Example request:
+        {
+            "jsonrpc": "2.0",
+            "method": "ApproveTx",
+            "params": [{
+                "transaction": {
+                    "to": "0xae967917c465db8578ca9024c205720b1a3651A9",
+                    "gas": "0x333",
+                    "gasPrice": "0x123",
+                    "value": "0x10",
+                    "data": "0xd7a5865800000000000000000000000000000000000000000000000000000000000000ff",
+                    "nonce": "0x0"
+                },
+                "from": "0xAe967917c465db8578ca9024c205720b1a3651A9",
+                "call_info": "Warning! Could not validate ABI-data against calldata\nSupplied ABI spec does not contain method signature in data: 0xd7a58658",
+                "meta": {
+                    "remote": "127.0.0.1:34572",
+                    "local": "localhost:8550",
+                    "scheme": "HTTP/1.1"
+                }
+            }],
+            "id": 1
+        }
+
+        :param transaction: transaction info
+        :param call_info: info abou the call, e.g. if ABI info could not be
+        :param meta: metadata about the request, e.g. where the call comes from
+        :return: 
+        """
+        transaction = req.get('transaction')
+        _from       = req.get('from')
+        call_info   = req.get('call_info')
+        meta        = req.get('meta')
+
+        return {
+            "approved" : False,
+            #"transaction" : transaction,
+  #          "from" : _from,
+#            "password" : None,
+        }
+
+    @public
+    def ApproveSignData(self, req):
+        """ Example request
+
+        """
+        return {"approved": False, "password" : None}
+
+    @public
+    def ApproveExport(self, req):
+        """ Example request
+
+        """
+        return {"approved" : False}
+
+    @public
+    def ApproveImport(self, req):
+        """ Example request
+
+        """
+        return { "approved" : False, "old_password": "", "new_password": ""}
+
+    @public
+    def ApproveListing(self, req):
+        """ Example request
+
+        """
+        return {'accounts': []}
+
+    @public
+    def ApproveNewAccount(self, req):
+        """
+        Example request
+
+        :return:
+        """
+        return {"approved": False,
+                #"password": ""
+                }
+
+    @public
+    def ShowError(self,message = {}):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowError'"},"id":1}
+
+        :param message: to show
+        :return: nothing
+        """
+        if 'text' in message.keys():
+            sys.stderr.write("Error: {}\n".format( message['text']))
+        return
+
+    @public
+    def ShowInfo(self,message = {}):
+        """
+        Example request
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowInfo'"},"id":0}
+
+        :param message: to display
+        :return:nothing
+        """
+
+        if 'text' in message.keys():
+            sys.stdout.write("Error: {}\n".format( message['text']))
+        return
+
+def main(args):
+
+    cmd = ["./clef", "--stdio-ui"]
+    if len(args) > 0 and args[0] == "test":
+        cmd.extend(["--stdio-ui-test"])
+    print("cmd: {}".format(" ".join(cmd)))
+    dispatcher = RPCDispatcher()
+    dispatcher.register_instance(StdIOHandler(), '')
+    # line buffered
+    p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    rpc_server = RPCServer(
+        PipeTransport(p.stdout, p.stdin),
+        JSONRPCProtocol(),
+        dispatcher
+    )
+    rpc_server.serve_forever()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

cmd/clef/rules.md

@@ -0,0 +1,236 @@
+# Rules
+
+The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
+
+It enables usecases like the following:
+
+* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
+* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`
+
+The two main features that are required for this to work well are;
+
+1. Rule Implementation: how to create, manage and interpret rules in a flexible but secure manner
+2. Credential managements and credentials; how to provide auto-unlock without exposing keys unnecessarily.
+
+The section below deals with both of them
+
+## Rule Implementation
+
+A ruleset file is implemented as a `js` file. Under the hood, the ruleset-engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
+defined in the UI protocol. Example:
+
+```javascript
+
+function asBig(str){
+    if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+    return new BigNumber(str)
+}
+
+// Approve transactions to a certain contract if value is below a certain limit
+function ApproveTx(req){
+
+    var limit = big.Newint("0xb1a2bc2ec50000")
+	var value = asBig(req.transaction.value);
+
+	if(req.transaction.to.toLowerCase()=="0xae967917c465db8578ca9024c205720b1a3651a9")
+	    && value.lt(limit) ){
+	    return "Approve"
+	 }
+    // If we return "Reject", it will be rejected.
+    // By not returning anything, it will be passed to the next UI, for manual processing
+}
+
+//Approve listings if request made from IPC
+function ApproveListing(req){
+    if (req.metadata.scheme == "ipc"){ return "Approve"}
+}
+
+```
+
+Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
+invokes the corresponding method. In doing so, there are three possible outcomes:
+
+1. JS returns "Approve"
+  * Auto-approve request
+2. JS returns "Reject"
+  * Auto-reject request
+3. Error occurs, or something else is returned
+  * Pass on to `next` ui: the regular UI channel.
+
+A more advanced example can be found below, "Example 1: ruleset for a rate-limited window", using `storage` to `Put` and `Get` `string`s by key.
+
+* At the time of writing, storage only exists as an ephemeral unencrypted implementation, to be used during testing.
+
+### Things to note
+
+The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
+
+* "use strict" will parse, but does nothing.
+* The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification.
+* Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported.
+
+Additionally, a few more have been added
+
+* The rule execution cannot load external javascript files.
+* The only preloaded libary is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
+* Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
+* Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
+* The JS engine has access to `storage` and `console`.
+
+#### Security considerations
+
+##### Security of ruleset
+
+Some security precautions can be made, such as:
+
+* Never load `ruleset.js` unless the file is `readonly` (`r-??-??-?`). If the user wishes to modify the ruleset, he must make it writeable and then set back to readonly.
+  * This is to prevent attacks where files are dropped on the users disk.
+* Since we're going to have to have some form of secure storage (not defined in this section), we could also store the `sha3` of the `ruleset.js` file in there.
+  * If the user wishes to modify the ruleset, he'd then have to perform e.g. `signer --attest /path/to/ruleset --credential <creds>`
+
+##### Security of implementation
+
+The drawbacks of this very flexible solution is that the `signer` needs to contain a javascript engine. This is pretty simple to implement, since it's already
+implemented for `geth`. There are no known security vulnerabilities in, nor have we had any security-problems with it so far.
+
+The javascript engine would be an added attack surface; but if the validation of `rulesets` is made good (with hash-based attestation), the actual javascript cannot be considered
+an attack surface -- if an attacker can control the ruleset, a much simpler attack would be to implement an "always-approve" rule instead of exploiting the js vm. The only benefit
+to be gained from attacking the actual `signer` process from the `js` side would be if it could somehow extract cryptographic keys from memory.
+
+##### Security in usability
+
+Javascript is flexible, but also easy to get wrong, especially when users assume that `js` can handle large integers natively. Typical errors
+include trying to multiply `gasCost` with `gas` without using `bigint`:s.
+
+It's unclear whether any other DSL could be more secure; since there's always the possibility of erroneously implementing a rule.
+
+
+## Credential management
+
+The ability to auto-approve transaction means that the signer needs to have necessary credentials to decrypt keyfiles. These passwords are hereafter called `ksp` (keystore pass).
+
+### Example implementation
+
+Upon startup of the signer, the signer is given a switch: `--seed <path/to/masterseed>`
+The `seed` contains a blob of bytes, which is the master seed for the `signer`.
+
+The `signer` uses the `seed` to:
+
+* Generate the `path` where the settings are stored.
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/vault.dat`
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/rules.js`
+* Generate the encryption password for `vault.dat`.
+
+The `vault.dat` would be an encrypted container storing the following information:
+
+* `ksp` entries
+* `sha256` hash of `rules.js`
+* Information about pair:ed callers (not yet specified)
+
+### Security considerations
+
+This would leave it up to the user to ensure that the `path/to/masterseed` is handled in a secure way. It's difficult to get around this, although one could
+imagine leveraging OS-level keychains where supported. The setup is however in general similar to how ssh-keys are  stored in `.ssh/`.
+
+
+# Implementation status
+
+This is now implemented (with ephemeral non-encrypted storage for now, so not yet enabled).
+
+## Example 1: ruleset for a rate-limited window
+
+
+```javascript
+
+	function big(str){
+		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str)
+	}
+
+	// Time window: 1 week
+	var window = 1000* 3600*24*7;
+
+	// Limit : 1 ether
+	var limit = new BigNumber("1e18");
+
+	function isLimitOk(transaction){
+		var value = big(transaction.value)
+		// Start of our window function
+		var windowstart = new Date().getTime() - window;
+
+		var txs = [];
+		var stored = storage.Get('txs');
+
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// First, remove all that have passed out of the time-window
+		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
+		console.log(txs, newtxs.length);
+
+		// Secondly, aggregate the current sum
+		sum = new BigNumber(0)
+
+		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+		console.log("ApproveTx > Sum so far", sum);
+		console.log("ApproveTx > Requested", value.toNumber());
+
+		// Would we exceed weekly limit ?
+		return sum.plus(value).lt(limit)
+
+	}
+	function ApproveTx(r){
+		if (isLimitOk(r.transaction)){
+			return "Approve"
+		}
+		return "Nope"
+	}
+
+	/**
+	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+ 	* 'response_str' contains the return value that will be sent to the external caller.
+	* The return value from this method is ignore - the reason for having this callback is to allow the
+	* ruleset to keep track of approved transactions.
+	*
+	* When implementing rate-limited rules, this callback should be used.
+	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+	* then accepts the transaction, this method will be called.
+	*
+	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+	*/
+ 	function OnApprovedTx(resp){
+		var value = big(resp.tx.value)
+		var txs = []
+		// Load stored transactions
+		var stored = storage.Get('txs');
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// Add this to the storage
+		txs.push({tstamp: new Date().getTime(), value: value});
+		storage.Put("txs", JSON.stringify(txs));
+	}
+
+```
+
+## Example 2: allow destination
+
+```javascript
+
+	function ApproveTx(r){
+		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
+		// Otherwise goes to manual processing
+	}
+
+```
+
+## Example 3: Allow listing
+
+```javascript
+
+    function ApproveListing(){
+        return "Approve"
+    }
+
+```

cmd/clef/tutorial.md

@@ -0,0 +1,198 @@
+## Initializing the signer
+
+First, initialize the master seed.
+
+```text
+#./signer init
+
+WARNING!
+
+The signer is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing.
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE!
+
+
+Enter 'ok' to proceed:
+>ok
+A master seed has been generated into /home/martin/.signer/secrets.dat
+
+This is required to be able to store credentials, such as :
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it.
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+```
+
+(for readability purposes, we'll remove the WARNING printout in the rest of this document)
+
+## Creating rules
+
+Now, you can create a rule-file.
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+```
+Get the `sha256` hash....
+```text
+#sha256sum rules.js
+6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
+```
+...And then `attest` the file:
+```text
+#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+
+INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+```
+At this point, we then start the signer with the rule-file:
+
+```text
+#./signer --rules rules.json
+
+INFO [02-21|12:15:18] Using CLI as UI-channel
+INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+DEBUG[02-21|12:15:18] Ledger support enabled
+DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+
+```
+
+Any list-requests will now be auto-approved by our rule-file.
+
+## Under the hood
+
+While doing the operations above, these files have been created:
+
+```text
+#ls -laR ~/.signer/
+/home/martin/.signer/:
+total 16
+drwx------  3 martin martin 4096 feb 21 12:14 .
+drwxr-xr-x 71 martin martin 4096 feb 21 12:12 ..
+drwx------  2 martin martin 4096 feb 21 12:14 43f73718397aa54d1b22
+-rwx------  1 martin martin  256 feb 21 12:12 secrets.dat
+
+/home/martin/.signer/43f73718397aa54d1b22:
+total 12
+drwx------ 2 martin martin 4096 feb 21 12:14 .
+drwx------ 3 martin martin 4096 feb 21 12:14 ..
+-rw------- 1 martin martin  159 feb 21 12:14 config.json
+
+#cat /home/martin/.signer/43f73718397aa54d1b22/config.json
+{"ruleset_sha256":{"iv":"6v4W4tfJxj3zZFbl","c":"6dt5RTDiTq93yh1qDEjpsat/tsKG7cb+vr3sza26IPL2fvsQ6ZoqFx++CPUa8yy6fD9Bbq41L01ehkKHTG3pOAeqTW6zc/+t0wv3AB6xPmU="}}
+
+```
+
+In `~/.signer`, the `secrets.dat` file was created, containing the `master_seed`.
+The `master_seed` was then used to derive a few other things:
+
+- `vault_location` : in this case `43f73718397aa54d1b22` .
+   - Thus, if you use a different `master_seed`, another `vault_location` will be used that does not conflict with each other.
+   - Example: `signer --signersecret /path/to/afile ...`
+- `config.json` which is the encrypted key/value storage for configuration data, containing the key `ruleset_sha256`.
+
+
+## Adding credentials
+
+In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+
+```text
+#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+
+INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
+```
+## More advanced rules
+
+Now let's update the rules to make use of credentials
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+function ApproveSignData(r){
+    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+    {
+        if(r.message.indexOf("bazonk") >= 0){
+            return "Approve"
+        }
+        return "Reject"
+    }
+    // Otherwise goes to manual processing
+}
+
+```
+In this example,
+* any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`,
+    * and auto-rejected if it does not.
+    * Any other signing-requests will be passed along for manual approve/reject.
+
+..attest the new file
+```text
+#sha256sum rules.js
+2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
+
+#./signer attest 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+
+INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+```
+
+And start the signer:
+
+```
+#./signer --rules rules.js
+
+INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[02-21|14:41:56] Trezor support enabled
+INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
+```
+And then test signing, once with `bazonk` and once without:
+
+```
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bazonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"result":"0x93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c"}
+
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"error":{"code":-32000,"message":"Request denied"}}
+
+```
+
+Meanwhile, in the signer output:
+```text
+INFO [02-21|14:42:41] Op approved
+INFO [02-21|14:42:56] Op rejected
+```
+
+The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
+
+```text
+#tail audit.log -n 4
+t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
+```

cmd/evm/runner.go

@@ -76,6 +76,7 @@ func runCmd(ctx *cli.Context) error {
 	logconfig := &vm.LogConfig{
 		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
 		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+		Debug:         ctx.GlobalBool(DebugFlag.Name),
 	}
 
 	var (
@@ -83,8 +84,8 @@ func runCmd(ctx *cli.Context) error {
 		debugLogger *vm.StructLogger
 		statedb     *state.StateDB
 		chainConfig *params.ChainConfig
-		sender      = common.StringToAddress("sender")
-		receiver    = common.StringToAddress("receiver")
+		sender      = common.BytesToAddress([]byte("sender"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -234,9 +235,7 @@ Gas used:           %d
 
 `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
 	}
-	if tracer != nil {
-		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime, err)
-	} else {
+	if tracer == nil {
 		fmt.Printf("0x%x\n", ret)
 		if err != nil {
 			fmt.Printf(" error: %v\n", err)

cmd/geth/bugcmd.go

@@ -49,15 +49,17 @@ func reportBug(ctx *cli.Context) error {
 	// execute template and write contents to buff
 	var buff bytes.Buffer
 
-	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "#### System information")
+	fmt.Fprintln(&buff)
 	fmt.Fprintln(&buff, "Version:", params.Version)
 	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
 	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
 	printOSDetails(&buff)
+	fmt.Fprintln(&buff, header)
 
 	// open a new GH issue
 	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
-		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+		fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String())
 	}
 	return nil
 }
@@ -97,13 +99,15 @@ func printCmdOut(w io.Writer, prefix, path string, args ...string) {
 	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
 }
 
-const header = `Please answer these questions before submitting your issue. Thanks!
+const header = `
+#### Expected behaviour
 
-#### What did you do?
- 
-#### What did you expect to see?
- 
-#### What did you see instead?
- 
-#### System details
+
+#### Actual behaviour
+
+
+#### Steps to reproduce the behaviour
+
+
+#### Backtrace
 `

cmd/geth/chaincmd.go

@@ -95,6 +95,34 @@ Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
 if already existing.`,
+	}
+	importPreimagesCommand = cli.Command{
+		Action:    utils.MigrateFlags(importPreimages),
+		Name:      "import-preimages",
+		Usage:     "Import the preimage database from an RLP stream",
+		ArgsUsage: "<datafile>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+		Description: `
+	The import-preimages command imports hash preimages from an RLP encoded stream.`,
+	}
+	exportPreimagesCommand = cli.Command{
+		Action:    utils.MigrateFlags(exportPreimages),
+		Name:      "export-preimages",
+		Usage:     "Export the preimage database into an RLP stream",
+		ArgsUsage: "<dumpfile>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+		Description: `
+The export-preimages command export hash preimages to an RLP encoded stream`,
 	}
 	copydbCommand = cli.Command{
 		Action:    utils.MigrateFlags(copyDb),
@@ -225,6 +253,13 @@ func importChain(ctx *cli.Context) error {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
+
+	ioStats, err := db.LDB().GetProperty("leveldb.iostats")
+	if err != nil {
+		utils.Fatalf("Failed to read database iostats: %v", err)
+	}
+	fmt.Println(ioStats)
+
 	fmt.Printf("Trie cache misses:  %d\n", trie.CacheMisses())
 	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
 
@@ -255,6 +290,12 @@ func importChain(ctx *cli.Context) error {
 	}
 	fmt.Println(stats)
 
+	ioStats, err = db.LDB().GetProperty("leveldb.iostats")
+	if err != nil {
+		utils.Fatalf("Failed to read database iostats: %v", err)
+	}
+	fmt.Println(ioStats)
+
 	return nil
 }
 
@@ -286,7 +327,39 @@ func exportChain(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
-	fmt.Printf("Export done in %v", time.Since(start))
+	fmt.Printf("Export done in %v\n", time.Since(start))
+	return nil
+}
+
+// importPreimages imports preimage data from the specified file.
+func importPreimages(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	stack := makeFullNode(ctx)
+	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+
+	start := time.Now()
+	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
+		utils.Fatalf("Export error: %v\n", err)
+	}
+	fmt.Printf("Export done in %v\n", time.Since(start))
+	return nil
+}
+
+// exportPreimages dumps the preimage data to specified json file in streaming way.
+func exportPreimages(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	stack := makeFullNode(ctx)
+	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+
+	start := time.Now()
+	if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
+		utils.Fatalf("Export error: %v\n", err)
+	}
+	fmt.Printf("Export done in %v\n", time.Since(start))
 	return nil
 }
 

cmd/geth/config.go

@@ -32,7 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	"github.com/naoina/toml"
 )
 

cmd/geth/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -46,8 +45,6 @@ const (
 var (
 	// Git SHA1 commit hash of the release (set via linker flags)
 	gitCommit = ""
-	// Ethereum address of the Geth release oracle.
-	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
 	// flags that configure the node
@@ -65,7 +62,6 @@ var (
 		utils.DashboardAddrFlag,
 		utils.DashboardPortFlag,
 		utils.DashboardRefreshFlag,
-		utils.DashboardAssetsFlag,
 		utils.EthashCacheDirFlag,
 		utils.EthashCachesInMemoryFlag,
 		utils.EthashCachesOnDiskFlag,
@@ -156,6 +152,8 @@ func init() {
 		initCommand,
 		importCommand,
 		exportCommand,
+		importPreimagesCommand,
+		exportPreimagesCommand,
 		copydbCommand,
 		removedbCommand,
 		dumpCommand,

cmd/puppeth/module_node.go

@@ -40,11 +40,11 @@ ADD genesis.json /genesis.json
 	ADD signer.pass /signer.pass
 {{end}}
 RUN \
-  echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
-	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+  echo 'geth --cache 512 init /genesis.json' > /root/geth.sh && \{{if .Unlock}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> /root/geth.sh && \{{end}}
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> /root/geth.sh
 
-ENTRYPOINT ["/bin/sh", "geth.sh"]
+ENTRYPOINT ["/bin/sh", "/root/geth.sh"]
 `
 
 // nodeComposefile is the docker-compose.yml file required to deploy and maintain

cmd/puppeth/module_wallet.go

@@ -37,7 +37,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'node server.js &'                     > wallet.sh && \
 	echo 'geth --cache 512 init /genesis.json' >> wallet.sh && \
-	echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*"' >> wallet.sh
+	echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh
 
 RUN \
 	sed -i 's/PuppethNetworkID/{{.NetworkID}}/g' dist/js/etherwallet-master.js && \

cmd/puppeth/puppeth.go

@@ -20,6 +20,7 @@ package main
 import (
 	"math/rand"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -34,7 +35,7 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:  "network",
-			Usage: "name of the network to administer",
+			Usage: "name of the network to administer (no spaces or hyphens, please)",
 		},
 		cli.IntFlag{
 			Name:  "loglevel",
@@ -47,6 +48,10 @@ func main() {
 		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
 		rand.Seed(time.Now().UnixNano())
 
+		network := c.String("network")
+		if strings.Contains(network, " ") || strings.Contains(network, "-") {
+			log.Crit("No spaces or hyphens allowed in network name")
+		}
 		// Start the wizard and relinquish control
 		makeWizard(c.String("network")).run()
 		return nil

cmd/utils/cmd.go

@@ -27,8 +27,11 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -105,6 +108,8 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 	}
 
 	log.Info("Importing blockchain", "file", fn)
+
+	// Open the file handle and potentially unwrap the gzip stream
 	fh, err := os.Open(fn)
 	if err != nil {
 		return err
@@ -180,8 +185,12 @@ func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block
 	return nil
 }
 
+// ExportChain exports a blockchain into the specified file, truncating any data
+// already present in the file.
 func ExportChain(blockchain *core.BlockChain, fn string) error {
 	log.Info("Exporting blockchain", "file", fn)
+
+	// Open the file handle and potentially wrap with a gzip stream
 	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
 	if err != nil {
 		return err
@@ -193,7 +202,7 @@ func ExportChain(blockchain *core.BlockChain, fn string) error {
 		writer = gzip.NewWriter(writer)
 		defer writer.(*gzip.Writer).Close()
 	}
-
+	// Iterate over the blocks and export them
 	if err := blockchain.Export(writer); err != nil {
 		return err
 	}
@@ -202,9 +211,12 @@ func ExportChain(blockchain *core.BlockChain, fn string) error {
 	return nil
 }
 
+// ExportAppendChain exports a blockchain into the specified file, appending to
+// the file if data already exists in it.
 func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, last uint64) error {
 	log.Info("Exporting blockchain", "file", fn)
-	// TODO verify mode perms
+
+	// Open the file handle and potentially wrap with a gzip stream
 	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
 	if err != nil {
 		return err
@@ -216,10 +228,86 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
 		writer = gzip.NewWriter(writer)
 		defer writer.(*gzip.Writer).Close()
 	}
-
+	// Iterate over the blocks and export them
 	if err := blockchain.ExportN(writer, first, last); err != nil {
 		return err
 	}
 	log.Info("Exported blockchain to", "file", fn)
 	return nil
 }
+
+// ImportPreimages imports a batch of exported hash preimages into the database.
+func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
+	log.Info("Importing preimages", "file", fn)
+
+	// Open the file handle and potentially unwrap the gzip stream
+	fh, err := os.Open(fn)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	var reader io.Reader = fh
+	if strings.HasSuffix(fn, ".gz") {
+		if reader, err = gzip.NewReader(reader); err != nil {
+			return err
+		}
+	}
+	stream := rlp.NewStream(reader, 0)
+
+	// Import the preimages in batches to prevent disk trashing
+	preimages := make(map[common.Hash][]byte)
+
+	for {
+		// Read the next entry and ensure it's not junk
+		var blob []byte
+
+		if err := stream.Decode(&blob); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		// Accumulate the preimages and flush when enough ws gathered
+		preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
+		if len(preimages) > 1024 {
+			if err := core.WritePreimages(db, 0, preimages); err != nil {
+				return err
+			}
+			preimages = make(map[common.Hash][]byte)
+		}
+	}
+	// Flush the last batch preimage data
+	if len(preimages) > 0 {
+		return core.WritePreimages(db, 0, preimages)
+	}
+	return nil
+}
+
+// ExportPreimages exports all known hash preimages into the specified file,
+// truncating any data already present in the file.
+func ExportPreimages(db *ethdb.LDBDatabase, fn string) error {
+	log.Info("Exporting preimages", "file", fn)
+
+	// Open the file handle and potentially wrap with a gzip stream
+	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	var writer io.Writer = fh
+	if strings.HasSuffix(fn, ".gz") {
+		writer = gzip.NewWriter(writer)
+		defer writer.(*gzip.Writer).Close()
+	}
+	// Iterate over the preimages and export them
+	it := db.NewIteratorWithPrefix([]byte("secure-key-"))
+	for it.Next() {
+		if err := rlp.Encode(writer, it.Value()); err != nil {
+			return err
+		}
+	}
+	log.Info("Exported preimages", "file", fn)
+	return nil
+}

cmd/utils/flags.go

@@ -55,7 +55,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -209,11 +209,6 @@ var (
 		Usage: "Dashboard metrics collection refresh rate",
 		Value: dashboard.DefaultConfig.Refresh,
 	}
-	DashboardAssetsFlag = cli.StringFlag{
-		Name:  "dashboard.assets",
-		Usage: "Developer flag to serve the dashboard from the local file system",
-		Value: dashboard.DefaultConfig.Assets,
-	}
 	// Ethash settings
 	EthashCacheDirFlag = DirectoryFlag{
 		Name:  "ethash.cachedir",
@@ -819,6 +814,9 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 
 	if ctx.GlobalIsSet(MaxPeersFlag.Name) {
 		cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
+		if lightServer && !ctx.GlobalIsSet(LightPeersFlag.Name) {
+			cfg.MaxPeers += lightPeers
+		}
 	} else {
 		if lightServer {
 			cfg.MaxPeers += lightPeers
@@ -1120,7 +1118,6 @@ func SetDashboardConfig(ctx *cli.Context, cfg *dashboard.Config) {
 	cfg.Host = ctx.GlobalString(DashboardAddrFlag.Name)
 	cfg.Port = ctx.GlobalInt(DashboardPortFlag.Name)
 	cfg.Refresh = ctx.GlobalDuration(DashboardRefreshFlag.Name)
-	cfg.Assets = ctx.GlobalString(DashboardAssetsFlag.Name)
 }
 
 // RegisterEthService adds an Ethereum client to the stack.

cmd/wnode/main.go

@@ -197,6 +197,8 @@ func initialize() {
 		if len(*argIP) == 0 {
 			argIP = scanLineA("Please enter your IP and port (e.g. 127.0.0.1:30348): ")
 		}
+	} else if *fileReader {
+		*bootstrapMode = true
 	} else {
 		if len(*argEnode) == 0 {
 			argEnode = scanLineA("Please enter the peer's enode: ")
@@ -205,11 +207,22 @@ func initialize() {
 		peers = append(peers, peer)
 	}
 
+	if *mailServerMode {
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+	}
+
 	cfg := &whisper.Config{
 		MaxMessageSize:     uint32(*argMaxSize),
 		MinimumAcceptedPOW: *argPoW,
 	}
 
+	shh = whisper.New(cfg)
+
 	if *argPoW != whisper.DefaultMinimumPoW {
 		err := shh.SetMinimumPoW(*argPoW)
 		if err != nil {
@@ -257,18 +270,8 @@ func initialize() {
 	}
 
 	if *mailServerMode {
-		if len(msPassword) == 0 {
-			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
-			if err != nil {
-				utils.Fatalf("Failed to read Mail Server password: %s", err)
-			}
-		}
-
-		shh = whisper.New(cfg)
 		shh.RegisterServer(&mailServer)
 		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
-	} else {
-		shh = whisper.New(cfg)
 	}
 
 	server = &p2p.Server{
@@ -306,7 +309,11 @@ func startServer() error {
 		configureNode()
 	}
 
-	if !*forwarderMode {
+	if *fileExMode {
+		fmt.Printf("Please type the file name to be send. To quit type: '%s'\n", quitCommand)
+	} else if *fileReader {
+		fmt.Printf("Please type the file name to be decrypted. To quit type: '%s'\n", quitCommand)
+	} else if !*forwarderMode {
 		fmt.Printf("Please type the message. To quit type: '%s'\n", quitCommand)
 	}
 	return nil
@@ -566,6 +573,7 @@ func sendMsg(payload []byte) common.Hash {
 	if err != nil {
 		utils.Fatalf("failed to create new message: %s", err)
 	}
+
 	envelope, err := msg.Wrap(&params)
 	if err != nil {
 		fmt.Printf("failed to seal message: %v \n", err)
@@ -601,15 +609,17 @@ func messageLoop() {
 			m2 := af.Retrieve()
 			messages := append(m1, m2...)
 			for _, msg := range messages {
+				reportedOnce := false
+				if !*fileExMode && len(msg.Payload) <= 2048 {
+					printMessageInfo(msg)
+					reportedOnce = true
+				}
+
 				// All messages are saved upon specifying argSaveDir.
 				// fileExMode only specifies how messages are displayed on the console after they are saved.
 				// if fileExMode == true, only the hashes are displayed, since messages might be too big.
 				if len(*argSaveDir) > 0 {
-					writeMessageToFile(*argSaveDir, msg)
-				}
-
-				if !*fileExMode && len(msg.Payload) <= 2048 {
-					printMessageInfo(msg)
+					writeMessageToFile(*argSaveDir, msg, !reportedOnce)
 				}
 			}
 		case <-done:
@@ -634,7 +644,11 @@ func printMessageInfo(msg *whisper.ReceivedMessage) {
 	}
 }
 
-func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
+func writeMessageToFile(dir string, msg *whisper.ReceivedMessage, show bool) {
+	if len(dir) == 0 {
+		return
+	}
+
 	timestamp := fmt.Sprintf("%d", msg.Sent)
 	name := fmt.Sprintf("%x", msg.EnvelopeHash)
 
@@ -643,22 +657,24 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
 		address = crypto.PubkeyToAddress(*msg.Src)
 	}
 
+	env := shh.GetEnvelope(msg.EnvelopeHash)
+	if env == nil {
+		fmt.Printf("\nUnexpected error: envelope not found: %x\n", msg.EnvelopeHash)
+		return
+	}
+
 	// this is a sample code; uncomment if you don't want to save your own messages.
 	//if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
 	//	fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name)
 	//	return
 	//}
 
-	if len(dir) > 0 {
-		fullpath := filepath.Join(dir, name)
-		err := ioutil.WriteFile(fullpath, msg.Raw, 0644)
-		if err != nil {
-			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
-		} else {
-			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw))
-		}
-	} else {
-		fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name)
+	fullpath := filepath.Join(dir, name)
+	err := ioutil.WriteFile(fullpath, env.Data, 0644)
+	if err != nil {
+		fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
+	} else if show {
+		fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(env.Data))
 	}
 }
 
@@ -682,18 +698,23 @@ func requestExpiredMessagesLoop() {
 	for {
 		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
 		timeUpp = scanUint("Please enter the upper limit of the time range (unix timestamp): ")
-		t = scanLine("Please enter the topic (hexadecimal): ")
-		if len(t) >= whisper.TopicLength*2 {
+		t = scanLine("Enter the topic (hex). Press enter to request all messages, regardless of the topic: ")
+		if len(t) == whisper.TopicLength*2 {
 			x, err := hex.DecodeString(t)
 			if err != nil {
-				utils.Fatalf("Failed to parse the topic: %s", err)
+				fmt.Printf("Failed to parse the topic: %s \n", err)
+				continue
 			}
 			xt = whisper.BytesToTopic(x)
 			bloom = whisper.TopicToBloom(xt)
 			obfuscateBloom(bloom)
-		} else {
+		} else if len(t) == 0 {
 			bloom = whisper.MakeFullNodeBloom()
+		} else {
+			fmt.Println("Error: topic is invalid, request aborted")
+			continue
 		}
+
 		if timeUpp == 0 {
 			timeUpp = 0xFFFFFFFF
 		}

common/compiler/solidity.go

@@ -65,7 +65,6 @@ type solcOutput struct {
 func (s *Solidity) makeArgs() []string {
 	p := []string{
 		"--combined-json", "bin,abi,userdoc,devdoc",
-		"--add-std",  // include standard lib contracts
 		"--optimize", // code optimizer switched on
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {

common/types.go

@@ -23,8 +23,10 @@ import (
 	"math/rand"
 	"reflect"
 
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"strings"
 )
 
 const (
@@ -45,9 +47,8 @@ func BytesToHash(b []byte) Hash {
 	h.SetBytes(b)
 	return h
 }
-func StringToHash(s string) Hash { return BytesToHash([]byte(s)) }
-func BigToHash(b *big.Int) Hash  { return BytesToHash(b.Bytes()) }
-func HexToHash(s string) Hash    { return BytesToHash(FromHex(s)) }
+func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) }
+func HexToHash(s string) Hash   { return BytesToHash(FromHex(s)) }
 
 // Get the string representation of the underlying hash
 func (h Hash) Str() string   { return string(h[:]) }
@@ -143,9 +144,8 @@ func BytesToAddress(b []byte) Address {
 	a.SetBytes(b)
 	return a
 }
-func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) }
-func BigToAddress(b *big.Int) Address  { return BytesToAddress(b.Bytes()) }
-func HexToAddress(s string) Address    { return BytesToAddress(FromHex(s)) }
+func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) }
+func HexToAddress(s string) Address   { return BytesToAddress(FromHex(s)) }
 
 // IsHexAddress verifies whether a string can represent a valid hex-encoded
 // Ethereum address or not.
@@ -240,3 +240,63 @@ func (a *UnprefixedAddress) UnmarshalText(input []byte) error {
 func (a UnprefixedAddress) MarshalText() ([]byte, error) {
 	return []byte(hex.EncodeToString(a[:])), nil
 }
+
+// MixedcaseAddress retains the original string, which may or may not be
+// correctly checksummed
+type MixedcaseAddress struct {
+	addr     Address
+	original string
+}
+
+// NewMixedcaseAddress constructor (mainly for testing)
+func NewMixedcaseAddress(addr Address) MixedcaseAddress {
+	return MixedcaseAddress{addr: addr, original: addr.Hex()}
+}
+
+// NewMixedcaseAddressFromString is mainly meant for unit-testing
+func NewMixedcaseAddressFromString(hexaddr string) (*MixedcaseAddress, error) {
+	if !IsHexAddress(hexaddr) {
+		return nil, fmt.Errorf("Invalid address")
+	}
+	a := FromHex(hexaddr)
+	return &MixedcaseAddress{addr: BytesToAddress(a), original: hexaddr}, nil
+}
+
+// UnmarshalJSON parses MixedcaseAddress
+func (ma *MixedcaseAddress) UnmarshalJSON(input []byte) error {
+	if err := hexutil.UnmarshalFixedJSON(addressT, input, ma.addr[:]); err != nil {
+		return err
+	}
+	return json.Unmarshal(input, &ma.original)
+}
+
+// MarshalJSON marshals the original value
+func (ma *MixedcaseAddress) MarshalJSON() ([]byte, error) {
+	if strings.HasPrefix(ma.original, "0x") || strings.HasPrefix(ma.original, "0X") {
+		return json.Marshal(fmt.Sprintf("0x%s", ma.original[2:]))
+	}
+	return json.Marshal(fmt.Sprintf("0x%s", ma.original))
+}
+
+// Address returns the address
+func (ma *MixedcaseAddress) Address() Address {
+	return ma.addr
+}
+
+// String implements fmt.Stringer
+func (ma *MixedcaseAddress) String() string {
+	if ma.ValidChecksum() {
+		return fmt.Sprintf("%s [chksum ok]", ma.original)
+	}
+	return fmt.Sprintf("%s [chksum INVALID]", ma.original)
+}
+
+// ValidChecksum returns true if the address has valid checksum
+func (ma *MixedcaseAddress) ValidChecksum() bool {
+	return ma.original == ma.addr.Hex()
+}
+
+// Original returns the mixed-case input string
+func (ma *MixedcaseAddress) Original() string {
+	return ma.original
+}

common/types_test.go

@@ -18,6 +18,7 @@ package common
 
 import (
 	"encoding/json"
+
 	"math/big"
 	"strings"
 	"testing"
@@ -149,3 +150,46 @@ func BenchmarkAddressHex(b *testing.B) {
 		testAddr.Hex()
 	}
 }
+
+func TestMixedcaseAccount_Address(t *testing.T) {
+
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md
+	// Note: 0X{checksum_addr} is not valid according to spec above
+
+	var res []struct {
+		A     MixedcaseAddress
+		Valid bool
+	}
+	if err := json.Unmarshal([]byte(`[
+		{"A" : "0xae967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0xAe967917c465db8578ca9024c205720b1a3651A9", "Valid": true},
+		{"A" : "0XAe967917c465db8578ca9024c205720b1a3651A9", "Valid": false},
+		{"A" : "0x1111111111111111111112222222222223333323", "Valid": true}
+		]`), &res); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, r := range res {
+		if got := r.A.ValidChecksum(); got != r.Valid {
+			t.Errorf("Expected checksum %v, got checksum %v, input %v", r.Valid, got, r.A.String())
+		}
+	}
+
+	//These should throw exceptions:
+	var r2 []MixedcaseAddress
+	for _, r := range []string{
+		`["0x11111111111111111111122222222222233333"]`,     // Too short
+		`["0x111111111111111111111222222222222333332"]`,    // Too short
+		`["0x11111111111111111111122222222222233333234"]`,  // Too long
+		`["0x111111111111111111111222222222222333332344"]`, // Too long
+		`["1111111111111111111112222222222223333323"]`,     // Missing 0x
+		`["x1111111111111111111112222222222223333323"]`,    // Missing 0
+		`["0xG111111111111111111112222222222223333323"]`,   //Non-hex
+	} {
+		if err := json.Unmarshal([]byte(r), &r2); err == nil {
+			t.Errorf("Expected failure, input %v", r)
+		}
+
+	}
+
+}

compression/rle/read_write.go

@@ -1,101 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package rle implements the run-length encoding used for Ethereum data.
-package rle
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-const (
-	token             byte = 0xfe
-	emptyShaToken          = 0xfd
-	emptyListShaToken      = 0xfe
-	tokenToken             = 0xff
-)
-
-var empty = crypto.Keccak256([]byte(""))
-var emptyList = crypto.Keccak256([]byte{0x80})
-
-func Decompress(dat []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-
-	for i := 0; i < len(dat); i++ {
-		if dat[i] == token {
-			if i+1 < len(dat) {
-				switch dat[i+1] {
-				case emptyShaToken:
-					buf.Write(empty)
-				case emptyListShaToken:
-					buf.Write(emptyList)
-				case tokenToken:
-					buf.WriteByte(token)
-				default:
-					buf.Write(make([]byte, int(dat[i+1]-2)))
-				}
-				i++
-			} else {
-				return nil, errors.New("error reading bytes. token encountered without proceeding bytes")
-			}
-		} else {
-			buf.WriteByte(dat[i])
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func compressChunk(dat []byte) (ret []byte, n int) {
-	switch {
-	case dat[0] == token:
-		return []byte{token, tokenToken}, 1
-	case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0:
-		j := 0
-		for j <= 254 && j < len(dat) {
-			if dat[j] != 0 {
-				break
-			}
-			j++
-		}
-		return []byte{token, byte(j + 2)}, j
-	case len(dat) >= 32:
-		if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) {
-			return []byte{token, emptyShaToken}, 32
-		} else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) {
-			return []byte{token, emptyListShaToken}, 32
-		}
-		fallthrough
-	default:
-		return dat[:1], 1
-	}
-}
-
-func Compress(dat []byte) []byte {
-	buf := new(bytes.Buffer)
-
-	i := 0
-	for i < len(dat) {
-		b, n := compressChunk(dat[i:])
-		buf.Write(b)
-		i += n
-	}
-
-	return buf.Bytes()
-}

compression/rle/read_write_test.go

@@ -1,50 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package rle
-
-import (
-	"testing"
-
-	checker "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { checker.TestingT(t) }
-
-type CompressionRleSuite struct{}
-
-var _ = checker.Suite(&CompressionRleSuite{})
-
-func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) {
-	exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70}
-	res, err := Decompress([]byte{token, 0xfd})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21}
-	res, err = Decompress([]byte{token, 0xfe})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	res, err = Decompress([]byte{token, 0xff})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, []byte{token})
-
-	res, err = Decompress([]byte{token, 12})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, make([]byte, 10))
-
-}

containers/vagrant/.gitignore

@@ -1 +0,0 @@
-.vagrant

containers/vagrant/Vagrantfile

@@ -1,38 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-require 'yaml'
-
-VAGRANTFILE_API_VERSION = 2
-VM_RAM = 2048
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-
-  config.vm.define "ubuntu", :primary => true do |ubuntu| 
-    ubuntu.vm.box = "ubuntu/trusty64"
-    ubuntu.vm.provision "shell", :path => "provisioners/shell/ubuntu.sh"
-  end
-
-  config.vm.define "debian", :primary => true do |debian|
-    debian.vm.box = "debian/jessie64"
-    debian.vm.provision "shell", :path => "provisioners/shell/debian.sh"
-  end
-  
-  config.vm.define "centos", :autostart => false do |centos| 
-    centos.vm.box = "centos/7"
-    centos.vm.provision "shell", :path => "provisioners/shell/centos.sh"
-  end
-  
-  config.vm.provider "virtualbox" do |vb|
-    vb.memory = VM_RAM
-  end
-
-  config.vm.provider "libvirt" do |lv|
-    lv.memory = VM_RAM
-
-    config.vm.synced_folder ".", "/home/vagrant/sync", :disabled => true
-  end
-  
-  config.vm.synced_folder ".", "/vagrant", :disabled => true
-  config.vm.synced_folder "../../", "/home/vagrant/go/src/github.com/ethereum/go-ethereum"
-end

containers/vagrant/provisioners/shell/centos.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-sudo yum install -y git wget
-sudo yum update -y
-
-wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz
-
-GETH_PATH="~vagrant/go/src/github.com/ethereum/go-ethereum/build/bin/"
-
-echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc 

containers/vagrant/provisioners/shell/debian.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-sudo apt-get install -y build-essential git-all wget
-sudo apt-get update
-
-wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz
-
-GETH_PATH="~vagrant/go/src/github.com/ethereum/go-ethereum/build/bin/"
-
-echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc 

containers/vagrant/provisioners/shell/ubuntu.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-sudo apt-get install -y build-essential git-all wget
-sudo apt-get update
-
-wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz
-
-GETH_PATH="~vagrant/go/src/github.com/ethereum/go-ethereum/build/bin/"
-
-echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc 

core/asm/compiler.go

@@ -114,7 +114,7 @@ func (c *Compiler) Compile() (string, []error) {
 }
 
 // next returns the next token and increments the
-// posititon.
+// position.
 func (c *Compiler) next() token {
 	token := c.tokens[c.pos]
 	c.pos++
@@ -122,7 +122,7 @@ func (c *Compiler) next() token {
 }
 
 // compile line compiles a single line instruction e.g.
-// "push 1", "jump @labal".
+// "push 1", "jump @label".
 func (c *Compiler) compileLine() error {
 	n := c.next()
 	if n.typ != lineStart {

core/asm/lexer.go

@@ -48,7 +48,7 @@ const (
 	lineEnd                           // emitted when a line ends
 	invalidStatement                  // any invalid statement
 	element                           // any element during element parsing
-	label                             // label is emitted when a labal is found
+	label                             // label is emitted when a label is found
 	labelDef                          // label definition is emitted when a new label is found
 	number                            // number is emitted when a number is found
 	stringValue                       // stringValue is emitted when a string has been found

core/blockchain.go

@@ -723,10 +723,13 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
 }
 
 // SetReceiptsData computes all the non-consensus fields of the receipts
-func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts types.Receipts) {
+func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts types.Receipts) error {
 	signer := types.MakeSigner(config, block.Number())
 
 	transactions, logIndex := block.Transactions(), uint(0)
+	if len(transactions) != len(receipts) {
+		return errors.New("transaction and receipt count mismatch")
+	}
 
 	for j := 0; j < len(receipts); j++ {
 		// The transaction hash can be retrieved from the transaction itself
@@ -754,6 +757,7 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty
 			logIndex++
 		}
 	}
+	return nil
 }
 
 // InsertReceiptChain attempts to complete an already existing header chain with
@@ -794,7 +798,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			continue
 		}
 		// Compute all the non-consensus fields of the receipts
-		SetReceiptsData(bc.chainConfig, block, receipts)
+		if err := SetReceiptsData(bc.chainConfig, block, receipts); err != nil {
+			return i, fmt.Errorf("failed to set receipts data: %v", err)
+		}
 		// Write all the data out into the database
 		if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil {
 			return i, fmt.Errorf("failed to write block body: %v", err)

core/blockchain_test.go

@@ -1338,3 +1338,114 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		}
 	}
 }
+
+// Benchmarks large blocks with value transfers to non-existing accounts
+func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) {
+	var (
+		signer          = types.HomesteadSigner{}
+		testBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+		bankFunds       = big.NewInt(100000000000000000)
+		gspec           = Genesis{
+			Config: params.TestChainConfig,
+			Alloc: GenesisAlloc{
+				testBankAddress: {Balance: bankFunds},
+				common.HexToAddress("0xc0de"): {
+					Code:    []byte{0x60, 0x01, 0x50},
+					Balance: big.NewInt(0),
+				}, // push 1, pop
+			},
+			GasLimit: 100e6, // 100 M
+		}
+	)
+	// Generate the original common chain segment and the two competing forks
+	engine := ethash.NewFaker()
+	db, _ := ethdb.NewMemDatabase()
+	genesis := gspec.MustCommit(db)
+
+	blockGenerator := func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{1})
+		for txi := 0; txi < numTxs; txi++ {
+			uniq := uint64(i*numTxs + txi)
+			recipient := recipientFn(uniq)
+			//recipient := common.BigToAddress(big.NewInt(0).SetUint64(1337 + uniq))
+			tx, err := types.SignTx(types.NewTransaction(uniq, recipient, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testBankKey)
+			if err != nil {
+				b.Error(err)
+			}
+			block.AddTx(tx)
+		}
+	}
+
+	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, numBlocks, blockGenerator)
+	b.StopTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Import the shared chain and the original canonical one
+		diskdb, _ := ethdb.NewMemDatabase()
+		gspec.MustCommit(diskdb)
+
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+		if err != nil {
+			b.Fatalf("failed to create tester chain: %v", err)
+		}
+		b.StartTimer()
+		if _, err := chain.InsertChain(shared); err != nil {
+			b.Fatalf("failed to insert shared chain: %v", err)
+		}
+		b.StopTimer()
+		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
+			b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
+
+		}
+	}
+}
+func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}

core/chain_makers.go

@@ -82,11 +82,23 @@ func (b *BlockGen) SetExtra(data []byte) {
 // added. Notably, contract code relying on the BLOCKHASH instruction
 // will panic during execution.
 func (b *BlockGen) AddTx(tx *types.Transaction) {
+	b.AddTxWithChain(nil, tx)
+}
+
+// AddTxWithChain adds a transaction to the generated block. If no coinbase has
+// been set, the block's coinbase is set to the zero address.
+//
+// AddTxWithChain panics if the transaction cannot be executed. In addition to
+// the protocol-imposed limitations (gas limit, etc.), there are some
+// further limitations on the content of transactions that can be
+// added. If contract code relies on the BLOCKHASH instruction,
+// the block in chain will be returned.
+func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) {
 	if b.gasPool == nil {
 		b.SetCoinbase(common.Address{})
 	}
 	b.statedb.Prepare(tx.Hash(), common.Hash{}, len(b.txs))
-	receipt, _, err := ApplyTransaction(b.config, nil, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
+	receipt, _, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
 	if err != nil {
 		panic(err)
 	}

core/database_util_test.go

@@ -317,7 +317,7 @@ func TestLookupStorage(t *testing.T) {
 			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
 				t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
 			}
-			if tx.String() != txn.String() {
+			if tx.Hash() != txn.Hash() {
 				t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
 			}
 		}

core/evm.go

@@ -60,13 +60,26 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author
 
 // GetHashFn returns a GetHashFunc which retrieves header hashes by number
 func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash {
+	var cache map[uint64]common.Hash
+
 	return func(n uint64) common.Hash {
-		for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) {
-			if header.Number.Uint64() == n {
-				return header.Hash()
+		// If there's no hash cache yet, make one
+		if cache == nil {
+			cache = map[uint64]common.Hash{
+				ref.Number.Uint64() - 1: ref.ParentHash,
+			}
+		}
+		// Try to fulfill the request from the cache
+		if hash, ok := cache[n]; ok {
+			return hash
+		}
+		// Not cached, iterate the blocks and cache the hashes
+		for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) {
+			cache[header.Number.Uint64()-1] = header.ParentHash
+			if n == header.Number.Uint64()-1 {
+				return header.ParentHash
 			}
 		}
-
 		return common.Hash{}
 	}
 }

core/headerchain.go

@@ -23,6 +23,7 @@ import (
 	"math"
 	"math/big"
 	mrand "math/rand"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -32,7 +33,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/hashicorp/golang-lru"
-	"sync/atomic"
 )
 
 const (

core/state/database.go

@@ -26,7 +26,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-// Trie cache generation limit after which to evic trie nodes from memory.
+// Trie cache generation limit after which to evict trie nodes from memory.
 var MaxTrieCacheGen = uint16(120)
 
 const (
@@ -151,9 +151,6 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 		return cached.(int), nil
 	}
 	code, err := db.ContractCode(addrHash, codeHash)
-	if err == nil {
-		db.codeSizeCache.Add(codeHash, len(code))
-	}
 	return len(code), err
 }
 

core/state/dump.go

@@ -53,7 +53,7 @@ func (self *StateDB) RawDump() Dump {
 			panic(err)
 		}
 
-		obj := newObject(nil, common.BytesToAddress(addr), data, nil)
+		obj := newObject(nil, common.BytesToAddress(addr), data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,

core/state/journal.go

@@ -22,11 +22,67 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// journalEntry is a modification entry in the state change journal that can be
+// reverted on demand.
 type journalEntry interface {
-	undo(*StateDB)
+	// revert undoes the changes introduced by this journal entry.
+	revert(*StateDB)
+
+	// dirtied returns the Ethereum address modified by this journal entry.
+	dirtied() *common.Address
 }
 
-type journal []journalEntry
+// journal contains the list of state modifications applied since the last state
+// commit. These are tracked to be able to be reverted in case of an execution
+// exception or revertal request.
+type journal struct {
+	entries []journalEntry         // Current changes tracked by the journal
+	dirties map[common.Address]int // Dirty accounts and the number of changes
+}
+
+// newJournal create a new initialized journal.
+func newJournal() *journal {
+	return &journal{
+		dirties: make(map[common.Address]int),
+	}
+}
+
+// append inserts a new modification entry to the end of the change journal.
+func (j *journal) append(entry journalEntry) {
+	j.entries = append(j.entries, entry)
+	if addr := entry.dirtied(); addr != nil {
+		j.dirties[*addr]++
+	}
+}
+
+// revert undoes a batch of journalled modifications along with any reverted
+// dirty handling too.
+func (j *journal) revert(statedb *StateDB, snapshot int) {
+	for i := len(j.entries) - 1; i >= snapshot; i-- {
+		// Undo the changes made by the operation
+		j.entries[i].revert(statedb)
+
+		// Drop any dirty tracking induced by the change
+		if addr := j.entries[i].dirtied(); addr != nil {
+			if j.dirties[*addr]--; j.dirties[*addr] == 0 {
+				delete(j.dirties, *addr)
+			}
+		}
+	}
+	j.entries = j.entries[:snapshot]
+}
+
+// dirty explicitly sets an address to dirty, even if the change entries would
+// otherwise suggest it as clean. This method is an ugly hack to handle the RIPEMD
+// precompile consensus exception.
+func (j *journal) dirty(addr common.Address) {
+	j.dirties[addr]++
+}
+
+// length returns the current number of entries in the journal.
+func (j *journal) length() int {
+	return len(j.entries)
+}
 
 type (
 	// Changes to the account trie.
@@ -77,16 +133,24 @@ type (
 	}
 )
 
-func (ch createObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) revert(s *StateDB) {
 	delete(s.stateObjects, *ch.account)
 	delete(s.stateObjectsDirty, *ch.account)
 }
 
-func (ch resetObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch resetObjectChange) revert(s *StateDB) {
 	s.setStateObject(ch.prev)
 }
 
-func (ch suicideChange) undo(s *StateDB) {
+func (ch resetObjectChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch suicideChange) revert(s *StateDB) {
 	obj := s.getStateObject(*ch.account)
 	if obj != nil {
 		obj.suicided = ch.prev
@@ -94,38 +158,60 @@ func (ch suicideChange) undo(s *StateDB) {
 	}
 }
 
-var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
-
-func (ch touchChange) undo(s *StateDB) {
-	if !ch.prev && *ch.account != ripemd {
-		s.getStateObject(*ch.account).touched = ch.prev
-		if !ch.prevDirty {
-			delete(s.stateObjectsDirty, *ch.account)
-		}
-	}
+func (ch suicideChange) dirtied() *common.Address {
+	return ch.account
 }
 
-func (ch balanceChange) undo(s *StateDB) {
+var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
+
+func (ch touchChange) revert(s *StateDB) {
+}
+
+func (ch touchChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch balanceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setBalance(ch.prev)
 }
 
-func (ch nonceChange) undo(s *StateDB) {
+func (ch balanceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch nonceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setNonce(ch.prev)
 }
 
-func (ch codeChange) undo(s *StateDB) {
+func (ch nonceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch codeChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
 }
 
-func (ch storageChange) undo(s *StateDB) {
+func (ch codeChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch storageChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setState(ch.key, ch.prevalue)
 }
 
-func (ch refundChange) undo(s *StateDB) {
+func (ch storageChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch refundChange) revert(s *StateDB) {
 	s.refund = ch.prev
 }
 
-func (ch addLogChange) undo(s *StateDB) {
+func (ch refundChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addLogChange) revert(s *StateDB) {
 	logs := s.logs[ch.txhash]
 	if len(logs) == 1 {
 		delete(s.logs, ch.txhash)
@@ -135,6 +221,14 @@ func (ch addLogChange) undo(s *StateDB) {
 	s.logSize--
 }
 
-func (ch addPreimageChange) undo(s *StateDB) {
+func (ch addLogChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addPreimageChange) revert(s *StateDB) {
 	delete(s.preimages, ch.hash)
 }
+
+func (ch addPreimageChange) dirtied() *common.Address {
+	return nil
+}

core/state/state_object.go

@@ -85,9 +85,7 @@ type stateObject struct {
 	// during the "update" phase of the state transition.
 	dirtyCode bool // true if the code was updated
 	suicided  bool
-	touched   bool
 	deleted   bool
-	onDirty   func(addr common.Address) // Callback method to mark a state object newly dirty
 }
 
 // empty returns whether the account is considered empty.
@@ -105,7 +103,7 @@ type Account struct {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, data Account, onDirty func(addr common.Address)) *stateObject {
+func newObject(db *StateDB, address common.Address, data Account) *stateObject {
 	if data.Balance == nil {
 		data.Balance = new(big.Int)
 	}
@@ -119,7 +117,6 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a
 		data:          data,
 		cachedStorage: make(Storage),
 		dirtyStorage:  make(Storage),
-		onDirty:       onDirty,
 	}
 }
 
@@ -137,23 +134,17 @@ func (self *stateObject) setError(err error) {
 
 func (self *stateObject) markSuicided() {
 	self.suicided = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (c *stateObject) touch() {
-	c.db.journal = append(c.db.journal, touchChange{
-		account:   &c.address,
-		prev:      c.touched,
-		prevDirty: c.onDirty == nil,
+	c.db.journal.append(touchChange{
+		account: &c.address,
 	})
-	if c.onDirty != nil {
-		c.onDirty(c.Address())
-		c.onDirty = nil
+	if c.address == ripemd {
+		// Explicitly put it in the dirty-cache, which is otherwise generated from
+		// flattened journals.
+		c.db.journal.dirty(c.address)
 	}
-	c.touched = true
 }
 
 func (c *stateObject) getTrie(db Database) Trie {
@@ -195,7 +186,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash {
 
 // SetState updates a value in account storage.
 func (self *stateObject) SetState(db Database, key, value common.Hash) {
-	self.db.journal = append(self.db.journal, storageChange{
+	self.db.journal.append(storageChange{
 		account:  &self.address,
 		key:      key,
 		prevalue: self.GetState(db, key),
@@ -207,10 +198,6 @@ func (self *stateObject) setState(key, value common.Hash) {
 	self.cachedStorage[key] = value
 	self.dirtyStorage[key] = value
 
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // updateTrie writes cached storage modifications into the object's storage trie.
@@ -274,7 +261,7 @@ func (c *stateObject) SubBalance(amount *big.Int) {
 }
 
 func (self *stateObject) SetBalance(amount *big.Int) {
-	self.db.journal = append(self.db.journal, balanceChange{
+	self.db.journal.append(balanceChange{
 		account: &self.address,
 		prev:    new(big.Int).Set(self.data.Balance),
 	})
@@ -283,17 +270,13 @@ func (self *stateObject) SetBalance(amount *big.Int) {
 
 func (self *stateObject) setBalance(amount *big.Int) {
 	self.data.Balance = amount
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // Return the gas back to the origin. Used by the Virtual machine or Closures
 func (c *stateObject) ReturnGas(gas *big.Int) {}
 
-func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject {
-	stateObject := newObject(db, self.address, self.data, onDirty)
+func (self *stateObject) deepCopy(db *StateDB) *stateObject {
+	stateObject := newObject(db, self.address, self.data)
 	if self.trie != nil {
 		stateObject.trie = db.db.CopyTrie(self.trie)
 	}
@@ -333,7 +316,7 @@ func (self *stateObject) Code(db Database) []byte {
 
 func (self *stateObject) SetCode(codeHash common.Hash, code []byte) {
 	prevcode := self.Code(self.db.db)
-	self.db.journal = append(self.db.journal, codeChange{
+	self.db.journal.append(codeChange{
 		account:  &self.address,
 		prevhash: self.CodeHash(),
 		prevcode: prevcode,
@@ -345,14 +328,10 @@ func (self *stateObject) setCode(codeHash common.Hash, code []byte) {
 	self.code = code
 	self.data.CodeHash = codeHash[:]
 	self.dirtyCode = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) SetNonce(nonce uint64) {
-	self.db.journal = append(self.db.journal, nonceChange{
+	self.db.journal.append(nonceChange{
 		account: &self.address,
 		prev:    self.data.Nonce,
 	})
@@ -361,10 +340,6 @@ func (self *stateObject) SetNonce(nonce uint64) {
 
 func (self *stateObject) setNonce(nonce uint64) {
 	self.data.Nonce = nonce
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) CodeHash() []byte {

core/state/statedb.go

@@ -76,14 +76,14 @@ type StateDB struct {
 
 	// Journal of state modifications. This is the backbone of
 	// Snapshot and RevertToSnapshot.
-	journal        journal
+	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
 
 	lock sync.Mutex
 }
 
-// Create a new state from a given trie
+// Create a new state from a given trie.
 func New(root common.Hash, db Database) (*StateDB, error) {
 	tr, err := db.OpenTrie(root)
 	if err != nil {
@@ -96,6 +96,7 @@ func New(root common.Hash, db Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		logs:              make(map[common.Hash][]*types.Log),
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}, nil
 }
 
@@ -110,7 +111,7 @@ func (self *StateDB) Error() error {
 	return self.dbErr
 }
 
-// Reset clears out all emphemeral state objects from the state db, but keeps
+// Reset clears out all ephemeral state objects from the state db, but keeps
 // the underlying state trie to avoid reloading data for the next operations.
 func (self *StateDB) Reset(root common.Hash) error {
 	tr, err := self.db.OpenTrie(root)
@@ -131,7 +132,7 @@ func (self *StateDB) Reset(root common.Hash) error {
 }
 
 func (self *StateDB) AddLog(log *types.Log) {
-	self.journal = append(self.journal, addLogChange{txhash: self.thash})
+	self.journal.append(addLogChange{txhash: self.thash})
 
 	log.TxHash = self.thash
 	log.BlockHash = self.bhash
@@ -156,7 +157,7 @@ func (self *StateDB) Logs() []*types.Log {
 // AddPreimage records a SHA3 preimage seen by the VM.
 func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) {
 	if _, ok := self.preimages[hash]; !ok {
-		self.journal = append(self.journal, addPreimageChange{hash: hash})
+		self.journal.append(addPreimageChange{hash: hash})
 		pi := make([]byte, len(preimage))
 		copy(pi, preimage)
 		self.preimages[hash] = pi
@@ -169,7 +170,7 @@ func (self *StateDB) Preimages() map[common.Hash][]byte {
 }
 
 func (self *StateDB) AddRefund(gas uint64) {
-	self.journal = append(self.journal, refundChange{prev: self.refund})
+	self.journal.append(refundChange{prev: self.refund})
 	self.refund += gas
 }
 
@@ -235,10 +236,10 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash {
 	return common.BytesToHash(stateObject.CodeHash())
 }
 
-func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) GetState(addr common.Address, bhash common.Hash) common.Hash {
+	stateObject := self.getStateObject(addr)
 	if stateObject != nil {
-		return stateObject.GetState(self.db, b)
+		return stateObject.GetState(self.db, bhash)
 	}
 	return common.Hash{}
 }
@@ -250,12 +251,12 @@ func (self *StateDB) Database() Database {
 
 // StorageTrie returns the storage trie of an account.
 // The return value is a copy and is nil for non-existent accounts.
-func (self *StateDB) StorageTrie(a common.Address) Trie {
-	stateObject := self.getStateObject(a)
+func (self *StateDB) StorageTrie(addr common.Address) Trie {
+	stateObject := self.getStateObject(addr)
 	if stateObject == nil {
 		return nil
 	}
-	cpy := stateObject.deepCopy(self, nil)
+	cpy := stateObject.deepCopy(self)
 	return cpy.updateTrie(self.db)
 }
 
@@ -271,7 +272,7 @@ func (self *StateDB) HasSuicided(addr common.Address) bool {
  * SETTERS
  */
 
-// AddBalance adds amount to the account associated with addr
+// AddBalance adds amount to the account associated with addr.
 func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -279,7 +280,7 @@ func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	}
 }
 
-// SubBalance subtracts amount from the account associated with addr
+// SubBalance subtracts amount from the account associated with addr.
 func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -308,7 +309,7 @@ func (self *StateDB) SetCode(addr common.Address, code []byte) {
 	}
 }
 
-func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
+func (self *StateDB) SetState(addr common.Address, key, value common.Hash) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
 		stateObject.SetState(self.db, key, value)
@@ -325,7 +326,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 	if stateObject == nil {
 		return false
 	}
-	self.journal = append(self.journal, suicideChange{
+	self.journal.append(suicideChange{
 		account:     &addr,
 		prev:        stateObject.suicided,
 		prevbalance: new(big.Int).Set(stateObject.Balance()),
@@ -337,7 +338,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 }
 
 //
-// Setting, updating & deleting state object methods
+// Setting, updating & deleting state object methods.
 //
 
 // updateStateObject writes the given object to the trie.
@@ -379,7 +380,7 @@ func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObje
 		return nil
 	}
 	// Insert into the live set.
-	obj := newObject(self, addr, data, self.MarkStateObjectDirty)
+	obj := newObject(self, addr, data)
 	self.setStateObject(obj)
 	return obj
 }
@@ -388,7 +389,7 @@ func (self *StateDB) setStateObject(object *stateObject) {
 	self.stateObjects[object.Address()] = object
 }
 
-// Retrieve a state object or create a new state object if nil
+// Retrieve a state object or create a new state object if nil.
 func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	stateObject := self.getStateObject(addr)
 	if stateObject == nil || stateObject.deleted {
@@ -397,22 +398,16 @@ func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	return stateObject
 }
 
-// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly
-// state object cache iteration to find a handful of modified ones.
-func (self *StateDB) MarkStateObjectDirty(addr common.Address) {
-	self.stateObjectsDirty[addr] = struct{}{}
-}
-
 // createObject creates a new state object. If there is an existing account with
 // the given address, it is overwritten and returned as the second return value.
 func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = self.getStateObject(addr)
-	newobj = newObject(self, addr, Account{}, self.MarkStateObjectDirty)
+	newobj = newObject(self, addr, Account{})
 	newobj.setNonce(0) // sets the object to dirty
 	if prev == nil {
-		self.journal = append(self.journal, createObjectChange{account: &addr})
+		self.journal.append(createObjectChange{account: &addr})
 	} else {
-		self.journal = append(self.journal, resetObjectChange{prev: prev})
+		self.journal.append(resetObjectChange{prev: prev})
 	}
 	self.setStateObject(newobj)
 	return newobj, prev
@@ -466,18 +461,35 @@ func (self *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                self.db,
 		trie:              self.db.CopyTrie(self.trie),
-		stateObjects:      make(map[common.Address]*stateObject, len(self.stateObjectsDirty)),
-		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
+		stateObjects:      make(map[common.Address]*stateObject, len(self.journal.dirties)),
+		stateObjectsDirty: make(map[common.Address]struct{}, len(self.journal.dirties)),
 		refund:            self.refund,
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}
 	// Copy the dirty states, logs, and preimages
-	for addr := range self.stateObjectsDirty {
-		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty)
-		state.stateObjectsDirty[addr] = struct{}{}
+	for addr := range self.journal.dirties {
+		// As documented [here](https://github.com/ethereum/go-ethereum/pull/16485#issuecomment-380438527),
+		// and in the Finalise-method, there is a case where an object is in the journal but not
+		// in the stateObjects: OOG after touch on ripeMD prior to Byzantium. Thus, we need to check for
+		// nil
+		if object, exist := self.stateObjects[addr]; exist {
+			state.stateObjects[addr] = object.deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
 	}
+	// Above, we don't copy the actual journal. This means that if the copy is copied, the
+	// loop above will be a no-op, since the copy's journal is empty.
+	// Thus, here we iterate over stateObjects, to enable copies of copies
+	for addr := range self.stateObjectsDirty {
+		if _, exist := state.stateObjects[addr]; !exist {
+			state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state)
+			state.stateObjectsDirty[addr] = struct{}{}
+		}
+	}
+
 	for hash, logs := range self.logs {
 		state.logs[hash] = make([]*types.Log, len(logs))
 		copy(state.logs[hash], logs)
@@ -492,7 +504,7 @@ func (self *StateDB) Copy() *StateDB {
 func (self *StateDB) Snapshot() int {
 	id := self.nextRevisionId
 	self.nextRevisionId++
-	self.validRevisions = append(self.validRevisions, revision{id, len(self.journal)})
+	self.validRevisions = append(self.validRevisions, revision{id, self.journal.length()})
 	return id
 }
 
@@ -507,13 +519,8 @@ func (self *StateDB) RevertToSnapshot(revid int) {
 	}
 	snapshot := self.validRevisions[idx].journalIndex
 
-	// Replay the journal to undo changes.
-	for i := len(self.journal) - 1; i >= snapshot; i-- {
-		self.journal[i].undo(self)
-	}
-	self.journal = self.journal[:snapshot]
-
-	// Remove invalidated snapshots from the stack.
+	// Replay the journal to undo changes and remove invalidated snapshots
+	self.journal.revert(self, snapshot)
 	self.validRevisions = self.validRevisions[:idx]
 }
 
@@ -525,14 +532,25 @@ func (self *StateDB) GetRefund() uint64 {
 // Finalise finalises the state by removing the self destructed objects
 // and clears the journal as well as the refunds.
 func (s *StateDB) Finalise(deleteEmptyObjects bool) {
-	for addr := range s.stateObjectsDirty {
-		stateObject := s.stateObjects[addr]
+	for addr := range s.journal.dirties {
+		stateObject, exist := s.stateObjects[addr]
+		if !exist {
+			// ripeMD is 'touched' at block 1714175, in tx 0x1237f737031e40bcde4a8b7e717b2d15e3ecadfe49bb1bbc71ee9deb09c6fcf2
+			// That tx goes out of gas, and although the notion of 'touched' does not exist there, the
+			// touch-event will still be recorded in the journal. Since ripeMD is a special snowflake,
+			// it will persist in the journal even though the journal is reverted. In this special circumstance,
+			// it may exist in `s.journal.dirties` but not in `s.stateObjects`.
+			// Thus, we can safely ignore it here
+			continue
+		}
+
 		if stateObject.suicided || (deleteEmptyObjects && stateObject.empty()) {
 			s.deleteStateObject(stateObject)
 		} else {
 			stateObject.updateRoot(s.db)
 			s.updateStateObject(stateObject)
 		}
+		s.stateObjectsDirty[addr] = struct{}{}
 	}
 	// Invalidate journal because reverting across transactions is not allowed.
 	s.clearJournalAndRefund()
@@ -576,7 +594,7 @@ func (s *StateDB) DeleteSuicides() {
 }
 
 func (s *StateDB) clearJournalAndRefund() {
-	s.journal = nil
+	s.journal = newJournal()
 	s.validRevisions = s.validRevisions[:0]
 	s.refund = 0
 }
@@ -585,6 +603,9 @@ func (s *StateDB) clearJournalAndRefund() {
 func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
+	for addr := range s.journal.dirties {
+		s.stateObjectsDirty[addr] = struct{}{}
+	}
 	// Commit objects to the trie.
 	for addr, stateObject := range s.stateObjects {
 		_, isDirty := s.stateObjectsDirty[addr]

core/state/statedb_test.go

@@ -413,11 +413,28 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 
 	snapshot := s.state.Snapshot()
 	s.state.AddBalance(common.Address{}, new(big.Int))
-	if len(s.state.stateObjectsDirty) != 1 {
+
+	if len(s.state.journal.dirties) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
 	s.state.RevertToSnapshot(snapshot)
-	if len(s.state.stateObjectsDirty) != 0 {
+	if len(s.state.journal.dirties) != 0 {
 		c.Fatal("expected no dirty state object")
 	}
 }
+
+// TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy.
+// See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512
+func TestCopyOfCopy(t *testing.T) {
+	db, _ := ethdb.NewMemDatabase()
+	sdb, _ := New(common.Hash{}, NewDatabase(db))
+	addr := common.HexToAddress("aaaa")
+	sdb.SetBalance(addr, big.NewInt(42))
+
+	if got := sdb.Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("1st copy fail, expected 42, got %v", got)
+	}
+	if got := sdb.Copy().Copy().GetBalance(addr).Uint64(); got != 42 {
+		t.Fatalf("2nd copy fail, expected 42, got %v", got)
+	}
+}

core/state_transition.go

@@ -132,28 +132,12 @@ func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, uint64, bool,
 	return NewStateTransition(evm, msg, gp).TransitionDb()
 }
 
-func (st *StateTransition) from() vm.AccountRef {
-	f := st.msg.From()
-	if !st.state.Exist(f) {
-		st.state.CreateAccount(f)
+// to returns the recipient of the message.
+func (st *StateTransition) to() common.Address {
+	if st.msg == nil || st.msg.To() == nil /* contract creation */ {
+		return common.Address{}
 	}
-	return vm.AccountRef(f)
-}
-
-func (st *StateTransition) to() vm.AccountRef {
-	if st.msg == nil {
-		return vm.AccountRef{}
-	}
-	to := st.msg.To()
-	if to == nil {
-		return vm.AccountRef{} // contract creation
-	}
-
-	reference := vm.AccountRef(*to)
-	if !st.state.Exist(*to) {
-		st.state.CreateAccount(*to)
-	}
-	return reference
+	return *st.msg.To()
 }
 
 func (st *StateTransition) useGas(amount uint64) error {
@@ -166,12 +150,8 @@ func (st *StateTransition) useGas(amount uint64) error {
 }
 
 func (st *StateTransition) buyGas() error {
-	var (
-		state  = st.state
-		sender = st.from()
-	)
 	mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.Gas()), st.gasPrice)
-	if state.GetBalance(sender.Address()).Cmp(mgval) < 0 {
+	if st.state.GetBalance(st.msg.From()).Cmp(mgval) < 0 {
 		return errInsufficientBalanceForGas
 	}
 	if err := st.gp.SubGas(st.msg.Gas()); err != nil {
@@ -180,20 +160,17 @@ func (st *StateTransition) buyGas() error {
 	st.gas += st.msg.Gas()
 
 	st.initialGas = st.msg.Gas()
-	state.SubBalance(sender.Address(), mgval)
+	st.state.SubBalance(st.msg.From(), mgval)
 	return nil
 }
 
 func (st *StateTransition) preCheck() error {
-	msg := st.msg
-	sender := st.from()
-
-	// Make sure this transaction's nonce is correct
-	if msg.CheckNonce() {
-		nonce := st.state.GetNonce(sender.Address())
-		if nonce < msg.Nonce() {
+	// Make sure this transaction's nonce is correct.
+	if st.msg.CheckNonce() {
+		nonce := st.state.GetNonce(st.msg.From())
+		if nonce < st.msg.Nonce() {
 			return ErrNonceTooHigh
-		} else if nonce > msg.Nonce() {
+		} else if nonce > st.msg.Nonce() {
 			return ErrNonceTooLow
 		}
 	}
@@ -208,8 +185,7 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		return
 	}
 	msg := st.msg
-	sender := st.from() // err checked in preCheck
-
+	sender := vm.AccountRef(msg.From())
 	homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber)
 	contractCreation := msg.To() == nil
 
@@ -233,8 +209,8 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		ret, _, st.gas, vmerr = evm.Create(sender, st.data, st.gas, st.value)
 	} else {
 		// Increment the nonce for the next transaction
-		st.state.SetNonce(sender.Address(), st.state.GetNonce(sender.Address())+1)
-		ret, st.gas, vmerr = evm.Call(sender, st.to().Address(), st.data, st.gas, st.value)
+		st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1)
+		ret, st.gas, vmerr = evm.Call(sender, st.to(), st.data, st.gas, st.value)
 	}
 	if vmerr != nil {
 		log.Debug("VM returned with error", "err", vmerr)
@@ -260,10 +236,8 @@ func (st *StateTransition) refundGas() {
 	st.gas += refund
 
 	// Return ETH for remaining gas, exchanged at the original rate.
-	sender := st.from()
-
 	remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gas), st.gasPrice)
-	st.state.AddBalance(sender.Address(), remaining)
+	st.state.AddBalance(st.msg.From(), remaining)
 
 	// Also return remaining gas to the block gas counter so it is
 	// available for the next transaction.

core/tx_list.go

@@ -367,9 +367,20 @@ func (l *txList) Flatten() types.Transactions {
 // price-sorted transactions to discard when the pool fills up.
 type priceHeap []*types.Transaction
 
-func (h priceHeap) Len() int           { return len(h) }
-func (h priceHeap) Less(i, j int) bool { return h[i].GasPrice().Cmp(h[j].GasPrice()) < 0 }
-func (h priceHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h priceHeap) Len() int      { return len(h) }
+func (h priceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+
+func (h priceHeap) Less(i, j int) bool {
+	// Sort primarily by price, returning the cheaper one
+	switch h[i].GasPrice().Cmp(h[j].GasPrice()) {
+	case -1:
+		return true
+	case 1:
+		return false
+	}
+	// If the prices match, stabilize via nonces (high nonce is worse)
+	return h[i].Nonce() > h[j].Nonce()
+}
 
 func (h *priceHeap) Push(x interface{}) {
 	*h = append(*h, x.(*types.Transaction))

core/tx_pool.go

@@ -320,7 +320,7 @@ func (pool *TxPool) loop() {
 				// Any non-locals old enough should be removed
 				if time.Since(pool.beats[addr]) > pool.config.Lifetime {
 					for _, tx := range pool.queue[addr].Flatten() {
-						pool.removeTx(tx.Hash())
+						pool.removeTx(tx.Hash(), true)
 					}
 				}
 			}
@@ -468,7 +468,7 @@ func (pool *TxPool) SetGasPrice(price *big.Int) {
 
 	pool.gasPrice = price
 	for _, tx := range pool.priced.Cap(price, pool.locals) {
-		pool.removeTx(tx.Hash())
+		pool.removeTx(tx.Hash(), false)
 	}
 	log.Info("Transaction pool price threshold updated", "price", price)
 }
@@ -630,7 +630,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		for _, tx := range drop {
 			log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice())
 			underpricedTxCounter.Inc(1)
-			pool.removeTx(tx.Hash())
+			pool.removeTx(tx.Hash(), false)
 		}
 	}
 	// If the transaction is replacing an already pending one, do directly
@@ -695,8 +695,10 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er
 		pool.priced.Removed()
 		queuedReplaceCounter.Inc(1)
 	}
-	pool.all[hash] = tx
-	pool.priced.Put(tx)
+	if pool.all[hash] == nil {
+		pool.all[hash] = tx
+		pool.priced.Put(tx)
+	}
 	return old != nil, nil
 }
 
@@ -862,7 +864,7 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction {
 
 // removeTx removes a single transaction from the queue, moving all subsequent
 // transactions back to the future queue.
-func (pool *TxPool) removeTx(hash common.Hash) {
+func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 	// Fetch the transaction we wish to delete
 	tx, ok := pool.all[hash]
 	if !ok {
@@ -872,20 +874,20 @@ func (pool *TxPool) removeTx(hash common.Hash) {
 
 	// Remove it from the list of known transactions
 	delete(pool.all, hash)
-	pool.priced.Removed()
-
+	if outofbound {
+		pool.priced.Removed()
+	}
 	// Remove the transaction from the pending lists and reset the account nonce
 	if pending := pool.pending[addr]; pending != nil {
 		if removed, invalids := pending.Remove(tx); removed {
-			// If no more transactions are left, remove the list
+			// If no more pending transactions are left, remove the list
 			if pending.Empty() {
 				delete(pool.pending, addr)
 				delete(pool.beats, addr)
-			} else {
-				// Otherwise postpone any invalidated transactions
-				for _, tx := range invalids {
-					pool.enqueueTx(tx.Hash(), tx)
-				}
+			}
+			// Postpone any invalidated transactions
+			for _, tx := range invalids {
+				pool.enqueueTx(tx.Hash(), tx)
 			}
 			// Update the account nonce if needed
 			if nonce := tx.Nonce(); pool.pendingState.GetNonce(addr) > nonce {
@@ -1053,7 +1055,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Drop all transactions if they are less than the overflow
 			if size := uint64(list.Len()); size <= drop {
 				for _, tx := range list.Flatten() {
-					pool.removeTx(tx.Hash())
+					pool.removeTx(tx.Hash(), true)
 				}
 				drop -= size
 				queuedRateLimitCounter.Inc(int64(size))
@@ -1062,7 +1064,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			// Otherwise drop only last few transactions
 			txs := list.Flatten()
 			for i := len(txs) - 1; i >= 0 && drop > 0; i-- {
-				pool.removeTx(txs[i].Hash())
+				pool.removeTx(txs[i].Hash(), true)
 				drop--
 				queuedRateLimitCounter.Inc(1)
 			}

core/tx_pool_test.go

@@ -209,15 +209,10 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
 
 	pool.lockedReset(nil, nil)
 
-	pendingTx, err := pool.Pending()
+	_, err := pool.Pending()
 	if err != nil {
 		t.Fatalf("Could not fetch pending transactions: %v", err)
 	}
-
-	for addr, txs := range pendingTx {
-		t.Logf("%0x: %d\n", addr, len(txs))
-	}
-
 	nonce = pool.State().GetNonce(address)
 	if nonce != 2 {
 		t.Fatalf("Invalid nonce, want 2, got %d", nonce)
@@ -350,7 +345,7 @@ func TestTransactionChainFork(t *testing.T) {
 	if _, err := pool.add(tx, false); err != nil {
 		t.Error("didn't expect error", err)
 	}
-	pool.removeTx(tx.Hash())
+	pool.removeTx(tx.Hash(), true)
 
 	// reset the pool's internal state
 	resetState()
@@ -557,74 +552,112 @@ func TestTransactionDropping(t *testing.T) {
 func TestTransactionPostponing(t *testing.T) {
 	t.Parallel()
 
-	// Create a test account and fund it
-	pool, key := setupTxPool()
+	// Create the pool to test the postponing with
+	db, _ := ethdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
+
+	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
 	defer pool.Stop()
 
-	account, _ := deriveSender(transaction(0, 0, key))
-	pool.currentState.AddBalance(account, big.NewInt(1000))
+	// Create two test accounts to produce different gap profiles with
+	keys := make([]*ecdsa.PrivateKey, 2)
+	accs := make([]common.Address, len(keys))
 
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		accs[i] = crypto.PubkeyToAddress(keys[i].PublicKey)
+
+		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(50100))
+	}
 	// Add a batch consecutive pending transactions for validation
-	txns := []*types.Transaction{}
-	for i := 0; i < 100; i++ {
-		var tx *types.Transaction
-		if i%2 == 0 {
-			tx = transaction(uint64(i), 100, key)
-		} else {
-			tx = transaction(uint64(i), 500, key)
+	txs := []*types.Transaction{}
+	for i, key := range keys {
+
+		for j := 0; j < 100; j++ {
+			var tx *types.Transaction
+			if (i+j)%2 == 0 {
+				tx = transaction(uint64(j), 25000, key)
+			} else {
+				tx = transaction(uint64(j), 50000, key)
+			}
+			txs = append(txs, tx)
+		}
+	}
+	for i, err := range pool.AddRemotes(txs) {
+		if err != nil {
+			t.Fatalf("tx %d: failed to add transactions: %v", i, err)
 		}
-		pool.promoteTx(account, tx.Hash(), tx)
-		txns = append(txns, tx)
 	}
 	// Check that pre and post validations leave the pool as is
-	if pool.pending[account].Len() != len(txns) {
-		t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns))
+	if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) {
+		t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs))
 	}
 	if len(pool.queue) != 0 {
-		t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 0)
+		t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0)
 	}
-	if len(pool.all) != len(txns) {
-		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns))
+	if len(pool.all) != len(txs) {
+		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs))
 	}
 	pool.lockedReset(nil, nil)
-	if pool.pending[account].Len() != len(txns) {
-		t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns))
+	if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) {
+		t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs))
 	}
 	if len(pool.queue) != 0 {
-		t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 0)
+		t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0)
 	}
-	if len(pool.all) != len(txns) {
-		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns))
+	if len(pool.all) != len(txs) {
+		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs))
 	}
 	// Reduce the balance of the account, and check that transactions are reorganised
-	pool.currentState.AddBalance(account, big.NewInt(-750))
+	for _, addr := range accs {
+		pool.currentState.AddBalance(addr, big.NewInt(-1))
+	}
 	pool.lockedReset(nil, nil)
 
-	if _, ok := pool.pending[account].txs.items[txns[0].Nonce()]; !ok {
-		t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txns[0])
+	// The first account's first transaction remains valid, check that subsequent
+	// ones are either filtered out, or queued up for later.
+	if _, ok := pool.pending[accs[0]].txs.items[txs[0].Nonce()]; !ok {
+		t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txs[0])
 	}
-	if _, ok := pool.queue[account].txs.items[txns[0].Nonce()]; ok {
-		t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txns[0])
+	if _, ok := pool.queue[accs[0]].txs.items[txs[0].Nonce()]; ok {
+		t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txs[0])
 	}
-	for i, tx := range txns[1:] {
+	for i, tx := range txs[1:100] {
 		if i%2 == 1 {
-			if _, ok := pool.pending[account].txs.items[tx.Nonce()]; ok {
+			if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok {
 				t.Errorf("tx %d: valid but future transaction present in pending pool: %v", i+1, tx)
 			}
-			if _, ok := pool.queue[account].txs.items[tx.Nonce()]; !ok {
+			if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; !ok {
 				t.Errorf("tx %d: valid but future transaction missing from future queue: %v", i+1, tx)
 			}
 		} else {
-			if _, ok := pool.pending[account].txs.items[tx.Nonce()]; ok {
+			if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok {
 				t.Errorf("tx %d: out-of-fund transaction present in pending pool: %v", i+1, tx)
 			}
-			if _, ok := pool.queue[account].txs.items[tx.Nonce()]; ok {
+			if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; ok {
 				t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", i+1, tx)
 			}
 		}
 	}
-	if len(pool.all) != len(txns)/2 {
-		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)/2)
+	// The second account's first transaction got invalid, check that all transactions
+	// are either filtered out, or queued up for later.
+	if pool.pending[accs[1]] != nil {
+		t.Errorf("invalidated account still has pending transactions")
+	}
+	for i, tx := range txs[100:] {
+		if i%2 == 1 {
+			if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; !ok {
+				t.Errorf("tx %d: valid but future transaction missing from future queue: %v", 100+i, tx)
+			}
+		} else {
+			if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; ok {
+				t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", 100+i, tx)
+			}
+		}
+	}
+	if len(pool.all) != len(txs)/2 {
+		t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)/2)
 	}
 }
 
@@ -949,11 +982,11 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) {
 	account2, _ := deriveSender(transaction(0, 0, key2))
 	pool2.currentState.AddBalance(account2, big.NewInt(1000000))
 
-	txns := []*types.Transaction{}
+	txs := []*types.Transaction{}
 	for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ {
-		txns = append(txns, transaction(origin+i, 100000, key2))
+		txs = append(txs, transaction(origin+i, 100000, key2))
 	}
-	pool2.AddRemotes(txns)
+	pool2.AddRemotes(txs)
 
 	// Ensure the batch optimization honors the same pool mechanics
 	if len(pool1.pending) != len(pool2.pending) {
@@ -1124,7 +1157,7 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	defer sub.Unsubscribe()
 
 	// Create a number of test accounts and fund them
-	keys := make([]*ecdsa.PrivateKey, 3)
+	keys := make([]*ecdsa.PrivateKey, 4)
 	for i := 0; i < len(keys); i++ {
 		keys[i], _ = crypto.GenerateKey()
 		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
@@ -1136,24 +1169,28 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	txs = append(txs, pricedTransaction(1, 100000, big.NewInt(1), keys[0]))
 	txs = append(txs, pricedTransaction(2, 100000, big.NewInt(2), keys[0]))
 
+	txs = append(txs, pricedTransaction(0, 100000, big.NewInt(1), keys[1]))
 	txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[1]))
-	txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[1]))
-	txs = append(txs, pricedTransaction(3, 100000, big.NewInt(2), keys[1]))
+	txs = append(txs, pricedTransaction(2, 100000, big.NewInt(2), keys[1]))
 
-	ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[2])
+	txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[2]))
+	txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[2]))
+	txs = append(txs, pricedTransaction(3, 100000, big.NewInt(2), keys[2]))
+
+	ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[3])
 
 	// Import the batch and that both pending and queued transactions match up
 	pool.AddRemotes(txs)
 	pool.AddLocal(ltx)
 
 	pending, queued := pool.Stats()
-	if pending != 4 {
-		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4)
+	if pending != 7 {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7)
 	}
 	if queued != 3 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3)
 	}
-	if err := validateEvents(events, 4); err != nil {
+	if err := validateEvents(events, 7); err != nil {
 		t.Fatalf("original event firing failed: %v", err)
 	}
 	if err := validateTxPoolInternals(pool); err != nil {
@@ -1166,8 +1203,8 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if pending != 2 {
 		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2)
 	}
-	if queued != 3 {
-		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3)
+	if queued != 5 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5)
 	}
 	if err := validateEvents(events, 0); err != nil {
 		t.Fatalf("reprice event firing failed: %v", err)
@@ -1179,7 +1216,10 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); err != ErrUnderpriced {
 		t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced {
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced {
+		t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
+	}
+	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); err != ErrUnderpriced {
 		t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
 	if err := validateEvents(events, 0); err != nil {
@@ -1189,7 +1229,7 @@ func TestTransactionPoolRepricing(t *testing.T) {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
 	// However we can add local underpriced transactions
-	tx := pricedTransaction(1, 100000, big.NewInt(1), keys[2])
+	tx := pricedTransaction(1, 100000, big.NewInt(1), keys[3])
 	if err := pool.AddLocal(tx); err != nil {
 		t.Fatalf("failed to add underpriced local transaction: %v", err)
 	}
@@ -1202,6 +1242,22 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	if err := validateTxPoolInternals(pool); err != nil {
 		t.Fatalf("pool internal state corrupted: %v", err)
 	}
+	// And we can fill gaps with properly priced transactions
+	if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil {
+		t.Fatalf("failed to add pending transaction: %v", err)
+	}
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil {
+		t.Fatalf("failed to add pending transaction: %v", err)
+	}
+	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil {
+		t.Fatalf("failed to add queued transaction: %v", err)
+	}
+	if err := validateEvents(events, 5); err != nil {
+		t.Fatalf("post-reprice event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
 }
 
 // Tests that setting the transaction pool gas price to a higher value does not
@@ -1327,13 +1383,13 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 		t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced)
 	}
 	// Ensure that adding high priced transactions drops cheap ones, but not own
-	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que -
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
-	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil {
+	if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3
 		t.Fatalf("failed to add well priced transaction: %v", err)
 	}
 	pending, queued = pool.Stats()
@@ -1343,7 +1399,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	if queued != 2 {
 		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
 	}
-	if err := validateEvents(events, 2); err != nil {
+	if err := validateEvents(events, 1); err != nil {
 		t.Fatalf("additional event firing failed: %v", err)
 	}
 	if err := validateTxPoolInternals(pool); err != nil {
@@ -1369,6 +1425,74 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	}
 }
 
+// Tests that more expensive transactions push out cheap ones from the pool, but
+// without producing instability by creating gaps that start jumping transactions
+// back and forth between queued/pending.
+func TestTransactionPoolStableUnderpricing(t *testing.T) {
+	t.Parallel()
+
+	// Create the pool to test the pricing enforcement with
+	db, _ := ethdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
+
+	config := testTxPoolConfig
+	config.GlobalSlots = 128
+	config.GlobalQueue = 0
+
+	pool := NewTxPool(config, params.TestChainConfig, blockchain)
+	defer pool.Stop()
+
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan TxPreEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
+	// Create a number of test accounts and fund them
+	keys := make([]*ecdsa.PrivateKey, 2)
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
+	}
+	// Fill up the entire queue with the same transaction price points
+	txs := types.Transactions{}
+	for i := uint64(0); i < config.GlobalSlots; i++ {
+		txs = append(txs, pricedTransaction(i, 100000, big.NewInt(1), keys[0]))
+	}
+	pool.AddRemotes(txs)
+
+	pending, queued := pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, int(config.GlobalSlots)); err != nil {
+		t.Fatalf("original event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+	// Ensure that adding high priced transactions drops a cheap, but doesn't produce a gap
+	if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil {
+		t.Fatalf("failed to add well priced transaction: %v", err)
+	}
+	pending, queued = pool.Stats()
+	if pending != int(config.GlobalSlots) {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, config.GlobalSlots)
+	}
+	if queued != 0 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0)
+	}
+	if err := validateEvents(events, 1); err != nil {
+		t.Fatalf("additional event firing failed: %v", err)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+}
+
 // Tests that the pool rejects replacement transactions that don't meet the minimum
 // price bump required.
 func TestTransactionReplacement(t *testing.T) {

core/types/block.go

@@ -19,7 +19,6 @@ package types
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 	"math/big"
 	"sort"
@@ -389,40 +388,6 @@ func (b *Block) Hash() common.Hash {
 	return v
 }
 
-func (b *Block) String() string {
-	str := fmt.Sprintf(`Block(#%v): Size: %v {
-MinerHash: %x
-%v
-Transactions:
-%v
-Uncles:
-%v
-}
-`, b.Number(), b.Size(), b.header.HashNoNonce(), b.header, b.transactions, b.uncles)
-	return str
-}
-
-func (h *Header) String() string {
-	return fmt.Sprintf(`Header(%x):
-[
-	ParentHash:	    %x
-	UncleHash:	    %x
-	Coinbase:	    %x
-	Root:		    %x
-	TxSha		    %x
-	ReceiptSha:	    %x
-	Bloom:		    %x
-	Difficulty:	    %v
-	Number:		    %v
-	GasLimit:	    %v
-	GasUsed:	    %v
-	Time:		    %v
-	Extra:		    %s
-	MixDigest:      %x
-	Nonce:		    %x
-]`, h.Hash(), h.ParentHash, h.UncleHash, h.Coinbase, h.Root, h.TxHash, h.ReceiptHash, h.Bloom, h.Difficulty, h.Number, h.GasLimit, h.GasUsed, h.Time, h.Extra, h.MixDigest, h.Nonce)
-}
-
 type Blocks []*Block
 
 type BlockBy func(b1, b2 *Block) bool

core/types/log.go

@@ -17,7 +17,6 @@
 package types
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -95,10 +94,6 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
-func (l *Log) String() string {
-	return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
-}
-
 // LogForStorage is a wrapper around a Log that flattens and parses the entire content of
 // a log including non-consensus fields.
 type LogForStorage Log

core/types/receipt.go

@@ -149,14 +149,6 @@ func (r *Receipt) Size() common.StorageSize {
 	return size
 }
 
-// String implements the Stringer interface.
-func (r *Receipt) String() string {
-	if len(r.PostState) == 0 {
-		return fmt.Sprintf("receipt{status=%d cgas=%v bloom=%x logs=%v}", r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs)
-	}
-	return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
-}
-
 // ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
 // entire content of a receipt, as opposed to only the consensus fields originally.
 type ReceiptForStorage Receipt

core/types/transaction.go

@@ -19,7 +19,6 @@ package types
 import (
 	"container/heap"
 	"errors"
-	"fmt"
 	"io"
 	"math/big"
 	"sync/atomic"
@@ -262,58 +261,6 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
 	return tx.data.V, tx.data.R, tx.data.S
 }
 
-func (tx *Transaction) String() string {
-	var from, to string
-	if tx.data.V != nil {
-		// make a best guess about the signer and use that to derive
-		// the sender.
-		signer := deriveSigner(tx.data.V)
-		if f, err := Sender(signer, tx); err != nil { // derive but don't cache
-			from = "[invalid sender: invalid sig]"
-		} else {
-			from = fmt.Sprintf("%x", f[:])
-		}
-	} else {
-		from = "[invalid sender: nil V field]"
-	}
-
-	if tx.data.Recipient == nil {
-		to = "[contract creation]"
-	} else {
-		to = fmt.Sprintf("%x", tx.data.Recipient[:])
-	}
-	enc, _ := rlp.EncodeToBytes(&tx.data)
-	return fmt.Sprintf(`
-	TX(%x)
-	Contract: %v
-	From:     %s
-	To:       %s
-	Nonce:    %v
-	GasPrice: %#x
-	GasLimit  %#x
-	Value:    %#x
-	Data:     0x%x
-	V:        %#x
-	R:        %#x
-	S:        %#x
-	Hex:      %x
-`,
-		tx.Hash(),
-		tx.data.Recipient == nil,
-		from,
-		to,
-		tx.data.AccountNonce,
-		tx.data.Price,
-		tx.data.GasLimit,
-		tx.data.Amount,
-		tx.data.Payload,
-		tx.data.V,
-		tx.data.R,
-		tx.data.S,
-		enc,
-	)
-}
-
 // Transactions is a Transaction slice type for basic sorting.
 type Transactions []*Transaction
 

core/vm/doc.go

@@ -17,19 +17,8 @@
 /*
 Package vm implements the Ethereum Virtual Machine.
 
-The vm package implements two EVMs, a byte code VM and a JIT VM. The BC
-(Byte Code) VM loops over a set of bytes and executes them according to the set
-of rules defined in the Ethereum yellow paper. When the BC VM is invoked it
-invokes the JIT VM in a separate goroutine and compiles the byte code in JIT
-instructions.
-
-The JIT VM, when invoked, loops around a set of pre-defined instructions until
-it either runs of gas, causes an internal error, returns or stops.
-
-The JIT optimiser attempts to pre-compile instructions in to chunks or segments
-such as multiple PUSH operations and static JUMPs. It does this by analysing the
-opcodes and attempts to match certain regions to known sets. Whenever the
-optimiser finds said segments it creates a new instruction and replaces the
-first occurrence in the sequence.
+The vm package implements one EVM, a byte code VM. The BC (Byte Code) VM loops
+over a set of bytes and executes them according to the set of rules defined
+in the Ethereum yellow paper.
 */
 package vm

core/vm/evm.go

@@ -111,7 +111,7 @@ type EVM struct {
 	callGasTemp uint64
 }
 
-// NewEVM retutrns a new EVM . The returned EVM is not thread safe and should
+// NewEVM returns a new EVM. The returned EVM is not thread safe and should
 // only ever be used *once*.
 func NewEVM(ctx Context, statedb StateDB, chainConfig *params.ChainConfig, vmConfig Config) *EVM {
 	evm := &EVM{
@@ -221,7 +221,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 		to       = AccountRef(caller.Address())
 	)
 	// initialise a new contract and set the code that is to be used by the
-	// E The contract is a scoped evmironment for this execution context
+	// EVM. The contract is a scoped environment for this execution context
 	// only.
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
@@ -341,7 +341,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 	evm.Transfer(evm.StateDB, caller.Address(), contractAddr, value)
 
 	// initialise a new contract and set the code that is to be used by the
-	// E The contract is a scoped evmironment for this execution context
+	// EVM. The contract is a scoped environment for this execution context
 	// only.
 	contract := NewContract(caller, AccountRef(contractAddr), value, gas)
 	contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code)

core/vm/instructions.go

@@ -30,6 +30,8 @@ import (
 
 var (
 	bigZero                  = new(big.Int)
+	tt255                    = math.BigPow(2, 255)
+	tt256                    = math.BigPow(2, 256)
 	errWriteProtection       = errors.New("evm: write protection")
 	errReturnDataOutOfBounds = errors.New("evm: return data out of bounds")
 	errExecutionReverted     = errors.New("evm: execution reverted")
@@ -37,20 +39,18 @@ var (
 )
 
 func opAdd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
-	stack.push(math.U256(x.Add(x, y)))
-
-	evm.interpreter.intPool.put(y)
+	x, y := stack.pop(), stack.peek()
+	math.U256(y.Add(x, y))
 
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opSub(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
-	stack.push(math.U256(x.Sub(x, y)))
-
-	evm.interpreter.intPool.put(y)
+	x, y := stack.pop(), stack.peek()
+	math.U256(y.Sub(x, y))
 
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
@@ -64,44 +64,39 @@ func opMul(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac
 }
 
 func opDiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
+	x, y := stack.pop(), stack.peek()
 	if y.Sign() != 0 {
-		stack.push(math.U256(x.Div(x, y)))
+		math.U256(y.Div(x, y))
 	} else {
-		stack.push(new(big.Int))
+		y.SetUint64(0)
 	}
-
-	evm.interpreter.intPool.put(y)
-
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opSdiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	x, y := math.S256(stack.pop()), math.S256(stack.pop())
-	if y.Sign() == 0 {
-		stack.push(new(big.Int))
-		return nil, nil
+	res := evm.interpreter.intPool.getZero()
+
+	if y.Sign() == 0 || x.Sign() == 0 {
+		stack.push(res)
 	} else {
-		n := new(big.Int)
-		if evm.interpreter.intPool.get().Mul(x, y).Sign() < 0 {
-			n.SetInt64(-1)
+		if x.Sign() != y.Sign() {
+			res.Div(x.Abs(x), y.Abs(y))
+			res.Neg(res)
 		} else {
-			n.SetInt64(1)
+			res.Div(x.Abs(x), y.Abs(y))
 		}
-
-		res := x.Div(x.Abs(x), y.Abs(y))
-		res.Mul(res, n)
-
 		stack.push(math.U256(res))
 	}
-	evm.interpreter.intPool.put(y)
+	evm.interpreter.intPool.put(x, y)
 	return nil, nil
 }
 
 func opMod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	x, y := stack.pop(), stack.pop()
 	if y.Sign() == 0 {
-		stack.push(new(big.Int))
+		stack.push(x.SetUint64(0))
 	} else {
 		stack.push(math.U256(x.Mod(x, y)))
 	}
@@ -111,23 +106,20 @@ func opMod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac
 
 func opSmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	x, y := math.S256(stack.pop()), math.S256(stack.pop())
+	res := evm.interpreter.intPool.getZero()
 
 	if y.Sign() == 0 {
-		stack.push(new(big.Int))
+		stack.push(res)
 	} else {
-		n := new(big.Int)
 		if x.Sign() < 0 {
-			n.SetInt64(-1)
+			res.Mod(x.Abs(x), y.Abs(y))
+			res.Neg(res)
 		} else {
-			n.SetInt64(1)
+			res.Mod(x.Abs(x), y.Abs(y))
 		}
-
-		res := x.Mod(x.Abs(x), y.Abs(y))
-		res.Mul(res, n)
-
 		stack.push(math.U256(res))
 	}
-	evm.interpreter.intPool.put(y)
+	evm.interpreter.intPool.put(x, y)
 	return nil, nil
 }
 
@@ -161,80 +153,99 @@ func opSignExtend(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stac
 }
 
 func opNot(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x := stack.pop()
-	stack.push(math.U256(x.Not(x)))
+	x := stack.peek()
+	math.U256(x.Not(x))
 	return nil, nil
 }
 
 func opLt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
+	x, y := stack.pop(), stack.peek()
 	if x.Cmp(y) < 0 {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
+		y.SetUint64(1)
 	} else {
-		stack.push(new(big.Int))
+		y.SetUint64(0)
 	}
-
-	evm.interpreter.intPool.put(x, y)
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opGt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
+	x, y := stack.pop(), stack.peek()
 	if x.Cmp(y) > 0 {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
+		y.SetUint64(1)
 	} else {
-		stack.push(new(big.Int))
+		y.SetUint64(0)
 	}
-
-	evm.interpreter.intPool.put(x, y)
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opSlt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := math.S256(stack.pop()), math.S256(stack.pop())
-	if x.Cmp(math.S256(y)) < 0 {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
-	} else {
-		stack.push(new(big.Int))
-	}
+	x, y := stack.pop(), stack.peek()
 
-	evm.interpreter.intPool.put(x, y)
+	xSign := x.Cmp(tt255)
+	ySign := y.Cmp(tt255)
+
+	switch {
+	case xSign >= 0 && ySign < 0:
+		y.SetUint64(1)
+
+	case xSign < 0 && ySign >= 0:
+		y.SetUint64(0)
+
+	default:
+		if x.Cmp(y) < 0 {
+			y.SetUint64(1)
+		} else {
+			y.SetUint64(0)
+		}
+	}
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opSgt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := math.S256(stack.pop()), math.S256(stack.pop())
-	if x.Cmp(y) > 0 {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
-	} else {
-		stack.push(new(big.Int))
-	}
+	x, y := stack.pop(), stack.peek()
 
-	evm.interpreter.intPool.put(x, y)
+	xSign := x.Cmp(tt255)
+	ySign := y.Cmp(tt255)
+
+	switch {
+	case xSign >= 0 && ySign < 0:
+		y.SetUint64(0)
+
+	case xSign < 0 && ySign >= 0:
+		y.SetUint64(1)
+
+	default:
+		if x.Cmp(y) > 0 {
+			y.SetUint64(1)
+		} else {
+			y.SetUint64(0)
+		}
+	}
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opEq(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
+	x, y := stack.pop(), stack.peek()
 	if x.Cmp(y) == 0 {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
+		y.SetUint64(1)
 	} else {
-		stack.push(new(big.Int))
+		y.SetUint64(0)
 	}
-
-	evm.interpreter.intPool.put(x, y)
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opIszero(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x := stack.pop()
+	x := stack.peek()
 	if x.Sign() > 0 {
-		stack.push(new(big.Int))
+		x.SetUint64(0)
 	} else {
-		stack.push(evm.interpreter.intPool.get().SetUint64(1))
+		x.SetUint64(1)
 	}
-
-	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
@@ -247,18 +258,18 @@ func opAnd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac
 }
 
 func opOr(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
-	stack.push(x.Or(x, y))
+	x, y := stack.pop(), stack.peek()
+	y.Or(x, y)
 
-	evm.interpreter.intPool.put(y)
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
 func opXor(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	x, y := stack.pop(), stack.pop()
-	stack.push(x.Xor(x, y))
+	x, y := stack.pop(), stack.peek()
+	y.Xor(x, y)
 
-	evm.interpreter.intPool.put(y)
+	evm.interpreter.intPool.put(x)
 	return nil, nil
 }
 
@@ -277,13 +288,12 @@ func opByte(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta
 func opAddmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	x, y, z := stack.pop(), stack.pop(), stack.pop()
 	if z.Cmp(bigZero) > 0 {
-		add := x.Add(x, y)
-		add.Mod(add, z)
-		stack.push(math.U256(add))
+		x.Add(x, y)
+		x.Mod(x, z)
+		stack.push(math.U256(x))
 	} else {
-		stack.push(new(big.Int))
+		stack.push(x.SetUint64(0))
 	}
-
 	evm.interpreter.intPool.put(y, z)
 	return nil, nil
 }
@@ -291,13 +301,12 @@ func opAddmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S
 func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	x, y, z := stack.pop(), stack.pop(), stack.pop()
 	if z.Cmp(bigZero) > 0 {
-		mul := x.Mul(x, y)
-		mul.Mod(mul, z)
-		stack.push(math.U256(mul))
+		x.Mul(x, y)
+		x.Mod(x, z)
+		stack.push(math.U256(x))
 	} else {
-		stack.push(new(big.Int))
+		stack.push(x.SetUint64(0))
 	}
-
 	evm.interpreter.intPool.put(y, z)
 	return nil, nil
 }
@@ -370,8 +379,7 @@ func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta
 	if evm.vmConfig.EnablePreimageRecording {
 		evm.StateDB.AddPreimage(common.BytesToHash(hash), data)
 	}
-
-	stack.push(new(big.Int).SetBytes(hash))
+	stack.push(evm.interpreter.intPool.get().SetBytes(hash))
 
 	evm.interpreter.intPool.put(offset, size)
 	return nil, nil
@@ -383,10 +391,8 @@ func opAddress(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *
 }
 
 func opBalance(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	addr := common.BigToAddress(stack.pop())
-	balance := evm.StateDB.GetBalance(addr)
-
-	stack.push(new(big.Int).Set(balance))
+	slot := stack.peek()
+	slot.Set(evm.StateDB.GetBalance(common.BigToAddress(slot)))
 	return nil, nil
 }
 
@@ -406,7 +412,7 @@ func opCallValue(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack
 }
 
 func opCallDataLoad(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	stack.push(new(big.Int).SetBytes(getDataBig(contract.Input, stack.pop(), big32)))
+	stack.push(evm.interpreter.intPool.get().SetBytes(getDataBig(contract.Input, stack.pop(), big32)))
 	return nil, nil
 }
 
@@ -437,10 +443,11 @@ func opReturnDataCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory,
 		memOffset  = stack.pop()
 		dataOffset = stack.pop()
 		length     = stack.pop()
-	)
-	defer evm.interpreter.intPool.put(memOffset, dataOffset, length)
 
-	end := new(big.Int).Add(dataOffset, length)
+		end = evm.interpreter.intPool.get().Add(dataOffset, length)
+	)
+	defer evm.interpreter.intPool.put(memOffset, dataOffset, length, end)
+
 	if end.BitLen() > 64 || uint64(len(evm.interpreter.returnData)) < end.Uint64() {
 		return nil, errReturnDataOutOfBounds
 	}
@@ -450,11 +457,8 @@ func opReturnDataCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory,
 }
 
 func opExtCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	a := stack.pop()
-
-	addr := common.BigToAddress(a)
-	a.SetInt64(int64(evm.StateDB.GetCodeSize(addr)))
-	stack.push(a)
+	slot := stack.peek()
+	slot.SetUint64(uint64(evm.StateDB.GetCodeSize(common.BigToAddress(slot))))
 
 	return nil, nil
 }
@@ -462,6 +466,7 @@ func opExtCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, sta
 func opCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	l := evm.interpreter.intPool.get().SetInt64(int64(len(contract.Code)))
 	stack.push(l)
+
 	return nil, nil
 }
 
@@ -504,9 +509,8 @@ func opBlockhash(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack
 	if num.Cmp(n) > 0 && num.Cmp(evm.BlockNumber) < 0 {
 		stack.push(evm.GetHash(num.Uint64()).Big())
 	} else {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	}
-
 	evm.interpreter.intPool.put(num, n)
 	return nil, nil
 }
@@ -517,22 +521,22 @@ func opCoinbase(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack
 }
 
 func opTimestamp(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	stack.push(math.U256(new(big.Int).Set(evm.Time)))
+	stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.Time)))
 	return nil, nil
 }
 
 func opNumber(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	stack.push(math.U256(new(big.Int).Set(evm.BlockNumber)))
+	stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.BlockNumber)))
 	return nil, nil
 }
 
 func opDifficulty(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	stack.push(math.U256(new(big.Int).Set(evm.Difficulty)))
+	stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.Difficulty)))
 	return nil, nil
 }
 
 func opGasLimit(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	stack.push(math.U256(new(big.Int).SetUint64(evm.GasLimit)))
+	stack.push(math.U256(evm.interpreter.intPool.get().SetUint64(evm.GasLimit)))
 	return nil, nil
 }
 
@@ -543,7 +547,7 @@ func opPop(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac
 
 func opMload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	offset := stack.pop()
-	val := new(big.Int).SetBytes(memory.Get(offset.Int64(), 32))
+	val := evm.interpreter.intPool.get().SetBytes(memory.Get(offset.Int64(), 32))
 	stack.push(val)
 
 	evm.interpreter.intPool.put(offset)
@@ -647,9 +651,9 @@ func opCreate(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S
 	// rule) and treat as an error, if the ruleset is frontier we must
 	// ignore this error and pretend the operation was successful.
 	if evm.ChainConfig().IsHomestead(evm.BlockNumber) && suberr == ErrCodeStoreOutOfGas {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else if suberr != nil && suberr != ErrCodeStoreOutOfGas {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else {
 		stack.push(addr.Big())
 	}
@@ -678,9 +682,9 @@ func opCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta
 	}
 	ret, returnGas, err := evm.Call(contract, toAddr, args, gas, value)
 	if err != nil {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else {
-		stack.push(big.NewInt(1))
+		stack.push(evm.interpreter.intPool.get().SetUint64(1))
 	}
 	if err == nil || err == errExecutionReverted {
 		memory.Set(retOffset.Uint64(), retSize.Uint64(), ret)
@@ -707,9 +711,9 @@ func opCallCode(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack
 	}
 	ret, returnGas, err := evm.CallCode(contract, toAddr, args, gas, value)
 	if err != nil {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else {
-		stack.push(big.NewInt(1))
+		stack.push(evm.interpreter.intPool.get().SetUint64(1))
 	}
 	if err == nil || err == errExecutionReverted {
 		memory.Set(retOffset.Uint64(), retSize.Uint64(), ret)
@@ -732,9 +736,9 @@ func opDelegateCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, st
 
 	ret, returnGas, err := evm.DelegateCall(contract, toAddr, args, gas)
 	if err != nil {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else {
-		stack.push(big.NewInt(1))
+		stack.push(evm.interpreter.intPool.get().SetUint64(1))
 	}
 	if err == nil || err == errExecutionReverted {
 		memory.Set(retOffset.Uint64(), retSize.Uint64(), ret)
@@ -757,9 +761,9 @@ func opStaticCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stac
 
 	ret, returnGas, err := evm.StaticCall(contract, toAddr, args, gas)
 	if err != nil {
-		stack.push(new(big.Int))
+		stack.push(evm.interpreter.intPool.getZero())
 	} else {
-		stack.push(big.NewInt(1))
+		stack.push(evm.interpreter.intPool.get().SetUint64(1))
 	}
 	if err == nil || err == errExecutionReverted {
 		memory.Set(retOffset.Uint64(), retSize.Uint64(), ret)

core/vm/instructions_test.go

@@ -32,7 +32,7 @@ type twoOperandTest struct {
 
 func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) {
 	var (
-		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
 		stack = newstack()
 		pc    = uint64(0)
 	)
@@ -68,7 +68,7 @@ func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64
 
 func TestByteOp(t *testing.T) {
 	var (
-		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
 		stack = newstack()
 	)
 	tests := []struct {
@@ -161,6 +161,7 @@ func TestSAR(t *testing.T) {
 
 func TestSGT(t *testing.T) {
 	tests := []twoOperandTest{
+
 		{"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
 		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
 		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
@@ -171,6 +172,8 @@ func TestSGT(t *testing.T) {
 		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
 		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
 		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "0000000000000000000000000000000000000000000000000000000000000000"},
 	}
 	testTwoOperandOp(t, tests, opSgt)
 }
@@ -187,13 +190,15 @@ func TestSLT(t *testing.T) {
 		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
 		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
 		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "0000000000000000000000000000000000000000000000000000000000000001"},
 	}
 	testTwoOperandOp(t, tests, opSlt)
 }
 
 func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) {
 	var (
-		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
 		stack = newstack()
 	)
 	// convert args
@@ -349,7 +354,11 @@ func BenchmarkOpEq(b *testing.B) {
 
 	opBenchmark(b, opEq, x, y)
 }
-
+func BenchmarkOpEq2(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201fffffffe"
+	opBenchmark(b, opEq, x, y)
+}
 func BenchmarkOpAnd(b *testing.B) {
 	x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff"
 	y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff"
@@ -412,3 +421,7 @@ func BenchmarkOpSAR(b *testing.B) {
 
 	opBenchmark(b, opSAR, x, y)
 }
+func BenchmarkOpIsZero(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	opBenchmark(b, opIszero, x)
+}

core/vm/interpreter.go

@@ -28,10 +28,6 @@ import (
 type Config struct {
 	// Debug enabled debugging Interpreter options
 	Debug bool
-	// EnableJit enabled the JIT VM
-	EnableJit bool
-	// ForceJit forces the JIT VM
-	ForceJit bool
 	// Tracer is the op code logger
 	Tracer Tracer
 	// NoRecursion disabled Interpreter call, callcode,
@@ -46,8 +42,8 @@ type Config struct {
 }
 
 // Interpreter is used to run Ethereum based contracts and will utilise the
-// passed evmironment to query external sources for state information.
-// The Interpreter will run the byte code VM or JIT VM based on the passed
+// passed environment to query external sources for state information.
+// The Interpreter will run the byte code VM based on the passed
 // configuration.
 type Interpreter struct {
 	evm      *EVM
@@ -188,7 +184,7 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 			}
 		}
 		// consume the gas and return an error if not enough gas is available.
-		// cost is explicitly set so that the capture state defer method cas get the proper cost
+		// cost is explicitly set so that the capture state defer method can get the proper cost
 		cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize)
 		if err != nil || !contract.UseGas(cost) {
 			return nil, ErrOutOfGas

core/vm/intpool.go

@@ -32,24 +32,36 @@ func newIntPool() *intPool {
 	return &intPool{pool: newstack()}
 }
 
+// get retrieves a big int from the pool, allocating one if the pool is empty.
+// Note, the returned int's value is arbitrary and will not be zeroed!
 func (p *intPool) get() *big.Int {
 	if p.pool.len() > 0 {
 		return p.pool.pop()
 	}
 	return new(big.Int)
 }
+
+// getZero retrieves a big int from the pool, setting it to zero or allocating
+// a new one if the pool is empty.
+func (p *intPool) getZero() *big.Int {
+	if p.pool.len() > 0 {
+		return p.pool.pop().SetUint64(0)
+	}
+	return new(big.Int)
+}
+
+// put returns an allocated big int to the pool to be later reused by get calls.
+// Note, the values as saved as is; neither put nor get zeroes the ints out!
 func (p *intPool) put(is ...*big.Int) {
 	if len(p.pool.data) > poolLimit {
 		return
 	}
-
 	for _, i := range is {
 		// verifyPool is a build flag. Pool verification makes sure the integrity
 		// of the integer pool by comparing values to a default value.
 		if verifyPool {
 			i.Set(checkVal)
 		}
-
 		p.pool.push(i)
 	}
 }

core/vm/logger.go

@@ -45,6 +45,7 @@ type LogConfig struct {
 	DisableMemory  bool // disable memory capture
 	DisableStack   bool // disable stack capture
 	DisableStorage bool // disable storage capture
+	Debug          bool // print output during capture end
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
@@ -184,6 +185,12 @@ func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost ui
 func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error {
 	l.output = output
 	l.err = err
+	if l.cfg.Debug {
+		fmt.Printf("0x%x\n", output)
+		if err != nil {
+			fmt.Printf(" error: %v\n", err)
+		}
+	}
 	return nil
 }
 

core/vm/logger_test.go

@@ -48,7 +48,7 @@ type dummyStateDB struct {
 
 func TestStoreCapture(t *testing.T) {
 	var (
-		env      = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		env      = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
 		logger   = NewStructLogger(nil)
 		mem      = NewMemory()
 		stack    = newstack()

core/vm/runtime/runtime.go

@@ -41,7 +41,6 @@ type Config struct {
 	GasLimit    uint64
 	GasPrice    *big.Int
 	Value       *big.Int
-	DisableJit  bool // "disable" so it's enabled by default
 	Debug       bool
 	EVMConfig   vm.Config
 
@@ -92,8 +91,7 @@ func setDefaults(cfg *Config) {
 // It returns the EVM's return value, the new state and an error if it failed.
 //
 // Executes sets up a in memory, temporarily, environment for the execution of
-// the given code. It enabled the JIT by default and make sure that it's restored
-// to it's original state afterwards.
+// the given code. It makes sure that it's restored to it's original state afterwards.
 func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 	if cfg == nil {
 		cfg = new(Config)
@@ -105,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db))
 	}
 	var (
-		address = common.StringToAddress("contract")
+		address = common.BytesToAddress([]byte("contract"))
 		vmenv   = NewEnv(cfg)
 		sender  = vm.AccountRef(cfg.Origin)
 	)
@@ -115,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 	// Call the code with the given configuration.
 	ret, _, err := vmenv.Call(
 		sender,
-		common.StringToAddress("contract"),
+		common.BytesToAddress([]byte("contract")),
 		input,
 		cfg.GasLimit,
 		cfg.Value,

core/vm/vm_jit.go

@@ -1,389 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// +build evmjit
-
-package vm
-
-/*
-
-void* evmjit_create();
-int   evmjit_run(void* _jit, void* _data, void* _env);
-void  evmjit_destroy(void* _jit);
-
-// Shared library evmjit (e.g. libevmjit.so) is expected to be installed in /usr/local/lib
-// More: https://github.com/ethereum/evmjit
-#cgo LDFLAGS: -levmjit
-*/
-import "C"
-
-/*
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"math/big"
-	"unsafe"
-
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-type JitVm struct {
-	env        EVM
-	me         ContextRef
-	callerAddr []byte
-	price      *big.Int
-	data       RuntimeData
-}
-
-type i256 [32]byte
-
-type RuntimeData struct {
-	gas          int64
-	gasPrice     int64
-	callData     *byte
-	callDataSize uint64
-	address      i256
-	caller       i256
-	origin       i256
-	callValue    i256
-	coinBase     i256
-	difficulty   i256
-	gasLimit     i256
-	number       uint64
-	timestamp    int64
-	code         *byte
-	codeSize     uint64
-	codeHash     i256
-}
-
-func hash2llvm(h []byte) i256 {
-	var m i256
-	copy(m[len(m)-len(h):], h) // right aligned copy
-	return m
-}
-
-func llvm2hash(m *i256) []byte {
-	return C.GoBytes(unsafe.Pointer(m), C.int(len(m)))
-}
-
-func llvm2hashRef(m *i256) []byte {
-	return (*[1 << 30]byte)(unsafe.Pointer(m))[:len(m):len(m)]
-}
-
-func address2llvm(addr []byte) i256 {
-	n := hash2llvm(addr)
-	bswap(&n)
-	return n
-}
-
-// bswap swap bytes of the 256-bit integer on LLVM side
-// TODO: Do not change memory on LLVM side, that can conflict with memory access optimizations
-func bswap(m *i256) *i256 {
-	for i, l := 0, len(m); i < l/2; i++ {
-		m[i], m[l-i-1] = m[l-i-1], m[i]
-	}
-	return m
-}
-
-func trim(m []byte) []byte {
-	skip := 0
-	for i := 0; i < len(m); i++ {
-		if m[i] == 0 {
-			skip++
-		} else {
-			break
-		}
-	}
-	return m[skip:]
-}
-
-func getDataPtr(m []byte) *byte {
-	var p *byte
-	if len(m) > 0 {
-		p = &m[0]
-	}
-	return p
-}
-
-func big2llvm(n *big.Int) i256 {
-	m := hash2llvm(n.Bytes())
-	bswap(&m)
-	return m
-}
-
-func llvm2big(m *i256) *big.Int {
-	n := big.NewInt(0)
-	for i := 0; i < len(m); i++ {
-		b := big.NewInt(int64(m[i]))
-		b.Lsh(b, uint(i)*8)
-		n.Add(n, b)
-	}
-	return n
-}
-
-// llvm2bytesRef creates a []byte slice that references byte buffer on LLVM side (as of that not controller by GC)
-// User must ensure that referenced memory is available to Go until the data is copied or not needed any more
-func llvm2bytesRef(data *byte, length uint64) []byte {
-	if length == 0 {
-		return nil
-	}
-	if data == nil {
-		panic("Unexpected nil data pointer")
-	}
-	return (*[1 << 30]byte)(unsafe.Pointer(data))[:length:length]
-}
-
-func untested(condition bool, message string) {
-	if condition {
-		panic("Condition `" + message + "` tested. Remove assert.")
-	}
-}
-
-func assert(condition bool, message string) {
-	if !condition {
-		panic("Assert `" + message + "` failed!")
-	}
-}
-
-func NewJitVm(env EVM) *JitVm {
-	return &JitVm{env: env}
-}
-
-func (self *JitVm) Run(me, caller ContextRef, code []byte, value, gas, price *big.Int, callData []byte) (ret []byte, err error) {
-	// TODO: depth is increased but never checked by VM. VM should not know about it at all.
-	self.env.SetDepth(self.env.Depth() + 1)
-
-	// TODO: Move it to Env.Call() or sth
-	if Precompiled[string(me.Address())] != nil {
-		// if it's address of precompiled contract
-		// fallback to standard VM
-		stdVm := New(self.env)
-		return stdVm.Run(me, caller, code, value, gas, price, callData)
-	}
-
-	if self.me != nil {
-		panic("JitVm.Run() can be called only once per JitVm instance")
-	}
-
-	self.me = me
-	self.callerAddr = caller.Address()
-	self.price = price
-
-	self.data.gas = gas.Int64()
-	self.data.gasPrice = price.Int64()
-	self.data.callData = getDataPtr(callData)
-	self.data.callDataSize = uint64(len(callData))
-	self.data.address = address2llvm(self.me.Address())
-	self.data.caller = address2llvm(caller.Address())
-	self.data.origin = address2llvm(self.env.Origin())
-	self.data.callValue = big2llvm(value)
-	self.data.coinBase = address2llvm(self.env.Coinbase())
-	self.data.difficulty = big2llvm(self.env.Difficulty())
-	self.data.gasLimit = big2llvm(self.env.GasLimit())
-	self.data.number = self.env.BlockNumber().Uint64()
-	self.data.timestamp = self.env.Time()
-	self.data.code = getDataPtr(code)
-	self.data.codeSize = uint64(len(code))
-	self.data.codeHash = hash2llvm(crypto.Keccak256(code)) // TODO: Get already computed hash?
-
-	jit := C.evmjit_create()
-	retCode := C.evmjit_run(jit, unsafe.Pointer(&self.data), unsafe.Pointer(self))
-
-	if retCode < 0 {
-		err = errors.New("OOG from JIT")
-		gas.SetInt64(0) // Set gas to 0, JIT does not bother
-	} else {
-		gas.SetInt64(self.data.gas)
-		if retCode == 1 { // RETURN
-			ret = C.GoBytes(unsafe.Pointer(self.data.callData), C.int(self.data.callDataSize))
-		} else if retCode == 2 { // SUICIDE
-			// TODO: Suicide support logic should be moved to Env to be shared by VM implementations
-			state := self.Env().State()
-			receiverAddr := llvm2hashRef(bswap(&self.data.address))
-			receiver := state.GetOrNewStateObject(receiverAddr)
-			balance := state.GetBalance(me.Address())
-			receiver.AddBalance(balance)
-			state.Delete(me.Address())
-		}
-	}
-
-	C.evmjit_destroy(jit)
-	return
-}
-
-func (self *JitVm) Printf(format string, v ...interface{}) VirtualMachine {
-	return self
-}
-
-func (self *JitVm) Endl() VirtualMachine {
-	return self
-}
-
-func (self *JitVm) Env() EVM {
-	return self.env
-}
-
-//export env_sha3
-func env_sha3(dataPtr *byte, length uint64, resultPtr unsafe.Pointer) {
-	data := llvm2bytesRef(dataPtr, length)
-	hash := crypto.Keccak256(data)
-	result := (*i256)(resultPtr)
-	*result = hash2llvm(hash)
-}
-
-//export env_sstore
-func env_sstore(vmPtr unsafe.Pointer, indexPtr unsafe.Pointer, valuePtr unsafe.Pointer) {
-	vm := (*JitVm)(vmPtr)
-	index := llvm2hash(bswap((*i256)(indexPtr)))
-	value := llvm2hash(bswap((*i256)(valuePtr)))
-	value = trim(value)
-	if len(value) == 0 {
-		prevValue := vm.env.State().GetState(vm.me.Address(), index)
-		if len(prevValue) != 0 {
-			vm.Env().State().Refund(vm.callerAddr, GasSStoreRefund)
-		}
-	}
-
-	vm.env.State().SetState(vm.me.Address(), index, value)
-}
-
-//export env_sload
-func env_sload(vmPtr unsafe.Pointer, indexPtr unsafe.Pointer, resultPtr unsafe.Pointer) {
-	vm := (*JitVm)(vmPtr)
-	index := llvm2hash(bswap((*i256)(indexPtr)))
-	value := vm.env.State().GetState(vm.me.Address(), index)
-	result := (*i256)(resultPtr)
-	*result = hash2llvm(value)
-	bswap(result)
-}
-
-//export env_balance
-func env_balance(_vm unsafe.Pointer, _addr unsafe.Pointer, _result unsafe.Pointer) {
-	vm := (*JitVm)(_vm)
-	addr := llvm2hash((*i256)(_addr))
-	balance := vm.Env().State().GetBalance(addr)
-	result := (*i256)(_result)
-	*result = big2llvm(balance)
-}
-
-//export env_blockhash
-func env_blockhash(_vm unsafe.Pointer, _number unsafe.Pointer, _result unsafe.Pointer) {
-	vm := (*JitVm)(_vm)
-	number := llvm2big((*i256)(_number))
-	result := (*i256)(_result)
-
-	currNumber := vm.Env().BlockNumber()
-	limit := big.NewInt(0).Sub(currNumber, big.NewInt(256))
-	if number.Cmp(limit) >= 0 && number.Cmp(currNumber) < 0 {
-		hash := vm.Env().GetHash(uint64(number.Int64()))
-		*result = hash2llvm(hash)
-	} else {
-		*result = i256{}
-	}
-}
-
-//export env_call
-func env_call(_vm unsafe.Pointer, _gas *int64, _receiveAddr unsafe.Pointer, _value unsafe.Pointer, inDataPtr unsafe.Pointer, inDataLen uint64, outDataPtr *byte, outDataLen uint64, _codeAddr unsafe.Pointer) bool {
-	vm := (*JitVm)(_vm)
-
-	//fmt.Printf("env_call (depth %d)\n", vm.Env().Depth())
-
-	defer func() {
-		if r := recover(); r != nil {
-			fmt.Printf("Recovered in env_call (depth %d, out %p %d): %s\n", vm.Env().Depth(), outDataPtr, outDataLen, r)
-		}
-	}()
-
-	balance := vm.Env().State().GetBalance(vm.me.Address())
-	value := llvm2big((*i256)(_value))
-
-	if balance.Cmp(value) >= 0 {
-		receiveAddr := llvm2hash((*i256)(_receiveAddr))
-		inData := C.GoBytes(inDataPtr, C.int(inDataLen))
-		outData := llvm2bytesRef(outDataPtr, outDataLen)
-		codeAddr := llvm2hash((*i256)(_codeAddr))
-		gas := big.NewInt(*_gas)
-		var out []byte
-		var err error
-		if bytes.Equal(codeAddr, receiveAddr) {
-			out, err = vm.env.Call(vm.me, codeAddr, inData, gas, vm.price, value)
-		} else {
-			out, err = vm.env.CallCode(vm.me, codeAddr, inData, gas, vm.price, value)
-		}
-		*_gas = gas.Int64()
-		if err == nil {
-			copy(outData, out)
-			return true
-		}
-	}
-
-	return false
-}
-
-//export env_create
-func env_create(_vm unsafe.Pointer, _gas *int64, _value unsafe.Pointer, initDataPtr unsafe.Pointer, initDataLen uint64, _result unsafe.Pointer) {
-	vm := (*JitVm)(_vm)
-
-	value := llvm2big((*i256)(_value))
-	initData := C.GoBytes(initDataPtr, C.int(initDataLen)) // TODO: Unnecessary if low balance
-	result := (*i256)(_result)
-	*result = i256{}
-
-	gas := big.NewInt(*_gas)
-	ret, suberr, ref := vm.env.Create(vm.me, nil, initData, gas, vm.price, value)
-	if suberr == nil {
-		dataGas := big.NewInt(int64(len(ret))) // TODO: Not the best design. env.Create can do it, it has the reference to gas counter
-		dataGas.Mul(dataGas, params.CreateDataGas)
-		gas.Sub(gas, dataGas)
-		*result = hash2llvm(ref.Address())
-	}
-	*_gas = gas.Int64()
-}
-
-//export env_log
-func env_log(_vm unsafe.Pointer, dataPtr unsafe.Pointer, dataLen uint64, _topic1 unsafe.Pointer, _topic2 unsafe.Pointer, _topic3 unsafe.Pointer, _topic4 unsafe.Pointer) {
-	vm := (*JitVm)(_vm)
-
-	data := C.GoBytes(dataPtr, C.int(dataLen))
-
-	topics := make([][]byte, 0, 4)
-	if _topic1 != nil {
-		topics = append(topics, llvm2hash((*i256)(_topic1)))
-	}
-	if _topic2 != nil {
-		topics = append(topics, llvm2hash((*i256)(_topic2)))
-	}
-	if _topic3 != nil {
-		topics = append(topics, llvm2hash((*i256)(_topic3)))
-	}
-	if _topic4 != nil {
-		topics = append(topics, llvm2hash((*i256)(_topic4)))
-	}
-
-	vm.Env().AddLog(state.NewLog(vm.me.Address(), topics, data, vm.env.BlockNumber().Uint64()))
-}
-
-//export env_extcode
-func env_extcode(_vm unsafe.Pointer, _addr unsafe.Pointer, o_size *uint64) *byte {
-	vm := (*JitVm)(_vm)
-	addr := llvm2hash((*i256)(_addr))
-	code := vm.Env().State().GetCode(addr)
-	*o_size = uint64(len(code))
-	return getDataPtr(code)
-}*/

crypto/bn256/bn256_fast.go

@@ -14,50 +14,22 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !amd64 appengine gccgo
+// +build amd64 arm64
 
 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve.
 package bn256
 
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/crypto/bn256/google"
-)
+import "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare"
 
 // G1 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G1 struct {
-	bn256.G1
-}
-
-// Add sets e to a+b and then returns e.
-func (e *G1) Add(a, b *G1) *G1 {
-	e.G1.Add(&a.G1, &b.G1)
-	return e
-}
-
-// ScalarMult sets e to a*k and then returns e.
-func (e *G1) ScalarMult(a *G1, k *big.Int) *G1 {
-	e.G1.ScalarMult(&a.G1, k)
-	return e
-}
+type G1 = bn256.G1
 
 // G2 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G2 struct {
-	bn256.G2
-}
+type G2 = bn256.G2
 
 // PairingCheck calculates the Optimal Ate pairing for a set of points.
 func PairingCheck(a []*G1, b []*G2) bool {
-	as := make([]*bn256.G1, len(a))
-	for i, p := range a {
-		as[i] = &p.G1
-	}
-	bs := make([]*bn256.G2, len(b))
-	for i, p := range b {
-		bs[i] = &p.G2
-	}
-	return bn256.PairingCheck(as, bs)
+	return bn256.PairingCheck(a, b)
 }

crypto/bn256/bn256_fuzz.go

@@ -0,0 +1,138 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build gofuzz
+
+package bn256
+
+import (
+	"bytes"
+	"math/big"
+
+	cloudflare "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare"
+	google "github.com/ethereum/go-ethereum/crypto/bn256/google"
+)
+
+// FuzzAdd fuzzez bn256 addition between the Google and Cloudflare libraries.
+func FuzzAdd(data []byte) int {
+	// Ensure we have enough data in the first place
+	if len(data) != 128 {
+		return 0
+	}
+	// Ensure both libs can parse the first curve point
+	xc := new(cloudflare.G1)
+	_, errc := xc.Unmarshal(data[:64])
+
+	xg := new(google.G1)
+	_, errg := xg.Unmarshal(data[:64])
+
+	if (errc == nil) != (errg == nil) {
+		panic("parse mismatch")
+	} else if errc != nil {
+		return 0
+	}
+	// Ensure both libs can parse the second curve point
+	yc := new(cloudflare.G1)
+	_, errc = yc.Unmarshal(data[64:])
+
+	yg := new(google.G1)
+	_, errg = yg.Unmarshal(data[64:])
+
+	if (errc == nil) != (errg == nil) {
+		panic("parse mismatch")
+	} else if errc != nil {
+		return 0
+	}
+	// Add the two points and ensure they result in the same output
+	rc := new(cloudflare.G1)
+	rc.Add(xc, yc)
+
+	rg := new(google.G1)
+	rg.Add(xg, yg)
+
+	if !bytes.Equal(rc.Marshal(), rg.Marshal()) {
+		panic("add mismatch")
+	}
+	return 0
+}
+
+// FuzzMul fuzzez bn256 scalar multiplication between the Google and Cloudflare
+// libraries.
+func FuzzMul(data []byte) int {
+	// Ensure we have enough data in the first place
+	if len(data) != 96 {
+		return 0
+	}
+	// Ensure both libs can parse the curve point
+	pc := new(cloudflare.G1)
+	_, errc := pc.Unmarshal(data[:64])
+
+	pg := new(google.G1)
+	_, errg := pg.Unmarshal(data[:64])
+
+	if (errc == nil) != (errg == nil) {
+		panic("parse mismatch")
+	} else if errc != nil {
+		return 0
+	}
+	// Add the two points and ensure they result in the same output
+	rc := new(cloudflare.G1)
+	rc.ScalarMult(pc, new(big.Int).SetBytes(data[64:]))
+
+	rg := new(google.G1)
+	rg.ScalarMult(pg, new(big.Int).SetBytes(data[64:]))
+
+	if !bytes.Equal(rc.Marshal(), rg.Marshal()) {
+		panic("scalar mul mismatch")
+	}
+	return 0
+}
+
+func FuzzPair(data []byte) int {
+	// Ensure we have enough data in the first place
+	if len(data) != 192 {
+		return 0
+	}
+	// Ensure both libs can parse the curve point
+	pc := new(cloudflare.G1)
+	_, errc := pc.Unmarshal(data[:64])
+
+	pg := new(google.G1)
+	_, errg := pg.Unmarshal(data[:64])
+
+	if (errc == nil) != (errg == nil) {
+		panic("parse mismatch")
+	} else if errc != nil {
+		return 0
+	}
+	// Ensure both libs can parse the twist point
+	tc := new(cloudflare.G2)
+	_, errc = tc.Unmarshal(data[64:])
+
+	tg := new(google.G2)
+	_, errg = tg.Unmarshal(data[64:])
+
+	if (errc == nil) != (errg == nil) {
+		panic("parse mismatch")
+	} else if errc != nil {
+		return 0
+	}
+	// Pair the two points and ensure thet result in the same output
+	if cloudflare.PairingCheck([]*cloudflare.G1{pc}, []*cloudflare.G2{tc}) != google.PairingCheck([]*google.G1{pg}, []*google.G2{tg}) {
+		panic("pair mismatch")
+	}
+	return 0
+}

crypto/bn256/bn256_slow.go

@@ -14,50 +14,22 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build amd64,!appengine,!gccgo
+// +build !amd64,!arm64
 
 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve.
 package bn256
 
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/crypto/bn256/cloudflare"
-)
+import "github.com/ethereum/go-ethereum/crypto/bn256/google"
 
 // G1 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G1 struct {
-	bn256.G1
-}
-
-// Add sets e to a+b and then returns e.
-func (e *G1) Add(a, b *G1) *G1 {
-	e.G1.Add(&a.G1, &b.G1)
-	return e
-}
-
-// ScalarMult sets e to a*k and then returns e.
-func (e *G1) ScalarMult(a *G1, k *big.Int) *G1 {
-	e.G1.ScalarMult(&a.G1, k)
-	return e
-}
+type G1 = bn256.G1
 
 // G2 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G2 struct {
-	bn256.G2
-}
+type G2 = bn256.G2
 
 // PairingCheck calculates the Optimal Ate pairing for a set of points.
 func PairingCheck(a []*G1, b []*G2) bool {
-	as := make([]*bn256.G1, len(a))
-	for i, p := range a {
-		as[i] = &p.G1
-	}
-	bs := make([]*bn256.G2, len(b))
-	for i, p := range b {
-		bs[i] = &p.G2
-	}
-	return bn256.PairingCheck(as, bs)
+	return bn256.PairingCheck(a, b)
 }

crypto/bn256/cloudflare/bn256_test.go

@@ -1,5 +1,3 @@
-// +build amd64,!appengine,!gccgo
-
 package bn256
 
 import (

crypto/bn256/cloudflare/curve.go

@@ -183,15 +183,24 @@ func (c *curvePoint) Double(a *curvePoint) {
 }
 
 func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int) {
-	sum, t := &curvePoint{}, &curvePoint{}
-	sum.SetInfinity()
+	precomp := [1 << 2]*curvePoint{nil, {}, {}, {}}
+	precomp[1].Set(a)
+	precomp[2].Set(a)
+	gfpMul(&precomp[2].x, &precomp[2].x, xiTo2PSquaredMinus2Over3)
+	precomp[3].Add(precomp[1], precomp[2])
 
-	for i := scalar.BitLen(); i >= 0; i-- {
+	multiScalar := curveLattice.Multi(scalar)
+
+	sum := &curvePoint{}
+	sum.SetInfinity()
+	t := &curvePoint{}
+
+	for i := len(multiScalar) - 1; i >= 0; i-- {
 		t.Double(sum)
-		if scalar.Bit(i) != 0 {
-			sum.Add(t, a)
-		} else {
+		if multiScalar[i] == 0 {
 			sum.Set(t)
+		} else {
+			sum.Add(t, precomp[multiScalar[i]])
 		}
 	}
 	c.Set(sum)

crypto/bn256/cloudflare/example_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build amd64,!appengine,!gccgo
-
 package bn256
 
 import (

crypto/bn256/cloudflare/gfp.h

@@ -1,32 +0,0 @@
-#define storeBlock(a0,a1,a2,a3, r) \
-	MOVQ a0,  0+r \
-	MOVQ a1,  8+r \
-	MOVQ a2, 16+r \
-	MOVQ a3, 24+r
-
-#define loadBlock(r, a0,a1,a2,a3) \
-	MOVQ  0+r, a0 \
-	MOVQ  8+r, a1 \
-	MOVQ 16+r, a2 \
-	MOVQ 24+r, a3
-
-#define gfpCarry(a0,a1,a2,a3,a4, b0,b1,b2,b3,b4) \
-	\ // b = a-p
-	MOVQ a0, b0 \
-	MOVQ a1, b1 \
-	MOVQ a2, b2 \
-	MOVQ a3, b3 \
-	MOVQ a4, b4 \
-	\
-	SUBQ ·p2+0(SB), b0 \
-	SBBQ ·p2+8(SB), b1 \
-	SBBQ ·p2+16(SB), b2 \
-	SBBQ ·p2+24(SB), b3 \
-	SBBQ $0, b4 \
-	\
-	\ // if b is negative then return a
-	\ // else return b
-	CMOVQCC b0, a0 \
-	CMOVQCC b1, a1 \
-	CMOVQCC b2, a2 \
-	CMOVQCC b3, a3

crypto/bn256/cloudflare/gfp_amd64.go

@@ -1,15 +0,0 @@
-// +build amd64,!appengine,!gccgo
-
-package bn256
-
-// go:noescape
-func gfpNeg(c, a *gfP)
-
-//go:noescape
-func gfpAdd(c, a, b *gfP)
-
-//go:noescape
-func gfpSub(c, a, b *gfP)
-
-//go:noescape
-func gfpMul(c, a, b *gfP)

crypto/bn256/cloudflare/gfp_amd64.s

@@ -1,8 +1,40 @@
-// +build amd64,!appengine,!gccgo
+// +build amd64,!generic
 
-#include "gfp.h"
-#include "mul.h"
-#include "mul_bmi2.h"
+#define storeBlock(a0,a1,a2,a3, r) \
+	MOVQ a0,  0+r \
+	MOVQ a1,  8+r \
+	MOVQ a2, 16+r \
+	MOVQ a3, 24+r
+
+#define loadBlock(r, a0,a1,a2,a3) \
+	MOVQ  0+r, a0 \
+	MOVQ  8+r, a1 \
+	MOVQ 16+r, a2 \
+	MOVQ 24+r, a3
+
+#define gfpCarry(a0,a1,a2,a3,a4, b0,b1,b2,b3,b4) \
+	\ // b = a-p
+	MOVQ a0, b0 \
+	MOVQ a1, b1 \
+	MOVQ a2, b2 \
+	MOVQ a3, b3 \
+	MOVQ a4, b4 \
+	\
+	SUBQ ·p2+0(SB), b0 \
+	SBBQ ·p2+8(SB), b1 \
+	SBBQ ·p2+16(SB), b2 \
+	SBBQ ·p2+24(SB), b3 \
+	SBBQ $0, b4 \
+	\
+	\ // if b is negative then return a
+	\ // else return b
+	CMOVQCC b0, a0 \
+	CMOVQCC b1, a1 \
+	CMOVQCC b2, a2 \
+	CMOVQCC b3, a3
+
+#include "mul_amd64.h"
+#include "mul_bmi2_amd64.h"
 
 TEXT ·gfpNeg(SB),0,$0-16
 	MOVQ ·p2+0(SB), R8

crypto/bn256/cloudflare/gfp_arm64.s

@@ -0,0 +1,113 @@
+// +build arm64,!generic
+
+#define storeBlock(a0,a1,a2,a3, r) \
+	MOVD a0,  0+r \
+	MOVD a1,  8+r \
+	MOVD a2, 16+r \
+	MOVD a3, 24+r
+
+#define loadBlock(r, a0,a1,a2,a3) \
+	MOVD  0+r, a0 \
+	MOVD  8+r, a1 \
+	MOVD 16+r, a2 \
+	MOVD 24+r, a3
+
+#define loadModulus(p0,p1,p2,p3) \
+	MOVD ·p2+0(SB), p0 \
+	MOVD ·p2+8(SB), p1 \
+	MOVD ·p2+16(SB), p2 \
+	MOVD ·p2+24(SB), p3
+
+#include "mul_arm64.h"
+
+TEXT ·gfpNeg(SB),0,$0-16
+	MOVD a+8(FP), R0
+	loadBlock(0(R0), R1,R2,R3,R4)
+	loadModulus(R5,R6,R7,R8)
+
+	SUBS R1, R5, R1
+	SBCS R2, R6, R2
+	SBCS R3, R7, R3
+	SBCS R4, R8, R4
+
+	SUBS R5, R1, R5
+	SBCS R6, R2, R6
+	SBCS R7, R3, R7
+	SBCS R8, R4, R8
+
+	CSEL CS, R5, R1, R1
+	CSEL CS, R6, R2, R2
+	CSEL CS, R7, R3, R3
+	CSEL CS, R8, R4, R4
+
+	MOVD c+0(FP), R0
+	storeBlock(R1,R2,R3,R4, 0(R0))
+	RET
+
+TEXT ·gfpAdd(SB),0,$0-24
+	MOVD a+8(FP), R0
+	loadBlock(0(R0), R1,R2,R3,R4)
+	MOVD b+16(FP), R0
+	loadBlock(0(R0), R5,R6,R7,R8)
+	loadModulus(R9,R10,R11,R12)
+	MOVD ZR, R0
+
+	ADDS R5, R1
+	ADCS R6, R2
+	ADCS R7, R3
+	ADCS R8, R4
+	ADCS ZR, R0
+
+	SUBS  R9, R1, R5
+	SBCS R10, R2, R6
+	SBCS R11, R3, R7
+	SBCS R12, R4, R8
+	SBCS  ZR, R0, R0
+
+	CSEL CS, R5, R1, R1
+	CSEL CS, R6, R2, R2
+	CSEL CS, R7, R3, R3
+	CSEL CS, R8, R4, R4
+
+	MOVD c+0(FP), R0
+	storeBlock(R1,R2,R3,R4, 0(R0))
+	RET
+
+TEXT ·gfpSub(SB),0,$0-24
+	MOVD a+8(FP), R0
+	loadBlock(0(R0), R1,R2,R3,R4)
+	MOVD b+16(FP), R0
+	loadBlock(0(R0), R5,R6,R7,R8)
+	loadModulus(R9,R10,R11,R12)
+
+	SUBS R5, R1
+	SBCS R6, R2
+	SBCS R7, R3
+	SBCS R8, R4
+
+	CSEL CS, ZR,  R9,  R9
+	CSEL CS, ZR, R10, R10
+	CSEL CS, ZR, R11, R11
+	CSEL CS, ZR, R12, R12
+
+	ADDS  R9, R1
+	ADCS R10, R2
+	ADCS R11, R3
+	ADCS R12, R4
+
+	MOVD c+0(FP), R0
+	storeBlock(R1,R2,R3,R4, 0(R0))
+	RET
+
+TEXT ·gfpMul(SB),0,$0-24
+	MOVD a+8(FP), R0
+	loadBlock(0(R0), R1,R2,R3,R4)
+	MOVD b+16(FP), R0
+	loadBlock(0(R0), R5,R6,R7,R8)
+
+	mul(R9,R10,R11,R12,R13,R14,R15,R16)
+	gfpReduce()
+
+	MOVD c+0(FP), R0
+	storeBlock(R1,R2,R3,R4, 0(R0))
+	RET