fix node/node.go flock dependency

les, light: verbose errors on state retrieval issues

.github/CODEOWNERS

@@ -2,31 +2,16 @@
 # Each line is a file pattern followed by one or more owners.
 
 accounts/usbwallet              @karalabe
+accounts/abi                    @gballet
 consensus                       @karalabe
 core/                           @karalabe @holiman
 eth/                            @karalabe
+graphql/                        @gballet
 les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe
 p2p/                            @fjl @zsfelfoldi
-swarm/bmt                       @zelig
-swarm/dev                       @lmars
-swarm/fuse                      @jmozah @holisticode
-swarm/grafana_dashboards        @nonsense
-swarm/metrics                   @nonsense @holisticode
-swarm/multihash                 @nolash
-swarm/network/bitvector         @zelig @janos @gbalint
-swarm/network/priorityqueue     @zelig @janos @gbalint
-swarm/network/simulations       @zelig
-swarm/network/stream            @janos @zelig @gbalint @holisticode @justelad
-swarm/network/stream/intervals  @janos
-swarm/network/stream/testing    @zelig
-swarm/pot                       @zelig
-swarm/pss                       @nolash @zelig @nonsense
-swarm/services                  @zelig
-swarm/state                     @justelad
-swarm/storage/encryption        @gbalint @zelig @nagydani
-swarm/storage/mock              @janos
-swarm/storage/mru               @nolash
-swarm/testutil                  @lmars
+p2p/simulations                 @zelig @nonsense @janos @justelad
+p2p/protocols                   @zelig @nonsense @janos @justelad
+p2p/testing                     @zelig @nonsense @janos @justelad
 whisper/                        @gballet @gluk256

.github/CONTRIBUTING.md

@@ -1,16 +1,40 @@
+# Contributing
+
+Thank you for considering to help out with the source code! We welcome 
+contributions from anyone on the internet, and are grateful for even the 
+smallest of fixes!
+
+If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+pull request for the maintainers to review and merge into the main code base. If
+you wish to submit more complex changes though, please check up with the core 
+devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+ensure those changes are in line with the general philosophy of the project 
+and/or get some early feedback which can make both your efforts much lighter as
+well as our review and merge procedures quick and simple.
+
+## Coding guidelines
+
+Please make sure your contributions adhere to our coding guidelines:
+
+ * Code must adhere to the official Go 
+[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines 
+(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go 
+[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+ * Pull requests need to be based on and opened against the `master` branch.
+ * Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
 ## Can I have feature X
 
-Before you do a feature request please check and make sure that it isn't possible
-through some other means. The JavaScript enabled console is a powerful feature
-in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+Before you submit a feature request, please check and make sure that it isn't 
+possible through some other means. The JavaScript-enabled console is a powerful 
+feature in the right hands. Please check our 
+[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
-## Contributing
+## Configuration, dependencies, and tests
 
-If you'd like to contribute to go-ethereum please fork, fix, commit and
-send a pull request. Commits which do not comply with the coding standards
-are ignored (use gofmt!).
-
-See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, testing, and
-dependency management.
+Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+for more details on configuring your environment, managing project dependencies
+and testing procedures.

.github/no-response.yml

@@ -1,11 +1,11 @@
 # Number of days of inactivity before an Issue is closed for lack of response
 daysUntilClose: 30
 # Label requiring a response
-responseRequiredLabel: more-information-needed
+responseRequiredLabel: "need:more-information"
 # Comment to post when closing an Issue for lack of response. Set to `false` to disable
 closeComment: >
   This issue has been automatically closed because there has been no response
   to our request for more information from the original author. With only the
   information that is currently in the issue, we don't have enough information
-  to take action. Please reach out if you have or find the answers we need so
-  that we can investigate further.
+  to take action. Please reach out if you have more relevant information or
+  answers to our questions so that we can investigate further.

.github/stale.yml

@@ -7,7 +7,7 @@ exemptLabels:
   - pinned
   - security
 # Label to use when marking an issue as stale
-staleLabel: stale
+staleLabel: "status:inactive"
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has not had

.gitignore

@@ -42,6 +42,7 @@ profile.cov
 /dashboard/assets/node_modules
 /dashboard/assets/stats.json
 /dashboard/assets/bundle.js
+/dashboard/assets/bundle.js.map
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log

.travis.yml

@@ -6,7 +6,18 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.9.x
+      go: 1.10.x
+      script:
+      - sudo modprobe fuse
+      - sudo chmod 666 /dev/fuse
+      - sudo chown root:$USER /etc/fuse.conf
+      - go run build/ci.go install
+      - go run build/ci.go test -coverage $TEST_PACKAGES
+
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.11.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -18,7 +29,7 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.12.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -27,8 +38,16 @@ matrix:
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.10.x
+      go: 1.12.x
       script:
+        - echo "Increase the maximum number of open file descriptors on macOS"
+        - NOFILE=20480
+        - sudo sysctl -w kern.maxfiles=$NOFILE
+        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+        - sudo launchctl limit maxfiles $NOFILE $NOFILE
+        - sudo launchctl limit maxfiles
+        - ulimit -S -n $NOFILE
+        - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
@@ -36,7 +55,7 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.12.x
       env:
         - lint
       git:
@@ -45,9 +64,10 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - os: linux
+    - if: repo = ethereum/go-ethereum AND type = push
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.12.x
       env:
         - ubuntu-ppa
       git:
@@ -59,14 +79,18 @@ matrix:
             - debhelper
             - dput
             - fakeroot
+            - python-bzrlib
+            - python-paramiko
       script:
-        - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
-    - os: linux
+    - if: repo = ethereum/go-ethereum AND type = push
+      os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.12.x
       env:
         - azure-linux
       git:
@@ -96,11 +120,12 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - os: linux
+    - if: repo = ethereum/go-ethereum AND type = push
+      os: linux
       dist: trusty
       services:
         - docker
-      go: 1.10.x
+      go: 1.12.x
       env:
         - azure-linux-mips
       git:
@@ -123,7 +148,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - os: linux
+    - if: repo = ethereum/go-ethereum AND type = push
+      os: linux
       dist: trusty
       addons:
         apt:
@@ -144,24 +170,24 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.3.linux-amd64.tar.gz | tar -xz
+        - curl https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip -o android-ndk-r17b.zip
-        - unzip -q android-ndk-r17b.zip && rm android-ndk-r17b.zip
-        - mv android-ndk-r17b $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r17b
+        - curl https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip -o android-ndk-r19b.zip
+        - unzip -q android-ndk-r19b.zip && rm android-ndk-r19b.zip
+        - mv android-ndk-r19b $ANDROID_HOME/ndk-bundle
 
         - mkdir -p $GOPATH/src/github.com/ethereum
-        - ln -s `pwd` $GOPATH/src/github.com/ethereum
+        - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - os: osx
-      go: 1.10.x
+    - if: repo = ethereum/go-ethereum AND type = push
+      os: osx
+      go: 1.12.x
       env:
         - azure-osx
         - azure-ios
@@ -188,9 +214,10 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - os: linux
+    - if: repo = ethereum/go-ethereum AND type = cron
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.12.x
       env:
         - azure-purge
       git:
@@ -198,9 +225,11 @@ matrix:
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
 
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
-    on_success: change
-    on_failure: always
+    - name: Race Detector for Swarm
+      if: repo = ethersphere/go-ethereum
+      os: linux
+      dist: trusty
+      go: 1.12.x
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm...

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Makefile

@@ -57,6 +57,9 @@ devtools:
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
+swarm-devtools:
+	env GOBIN= go install ./cmd/swarm/mimegen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

README.md

@@ -1,13 +1,13 @@
 ## Go Ethereum
 
-Official golang implementation of the Ethereum protocol.
+Official Golang implementation of the Ethereum protocol.
 
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
 Automated builds are available for stable releases and the unstable master branch.
 Binary archives are published at https://geth.ethereum.org/downloads/.
@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.10 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -35,12 +35,12 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | Command    | Description |
 |:----------:|-------------|
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
-| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
+| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
 | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
-| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
-| `swarm`    | Swarm daemon and tools. This is the entrypoint for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
+| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
+| `swarm`    | Swarm daemon and tools. This is the entry point for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
 | `puppeth`    | a CLI wizard that aids in creating a new Ethereum network. |
 
 ## Running geth
@@ -72,7 +72,7 @@ This command will:
    This tool is optional and if you leave it out you can always attach to an already running Geth instance
    with `geth attach`.
 
-### Full node on the Ethereum test network
+### A Full node on the Ethereum test network
 
 Transitioning towards developers, if you'd like to play around with creating Ethereum contracts, you
 almost certainly would like to do that without any real money involved until you get the hang of the
@@ -83,10 +83,10 @@ network with your node, which is fully equivalent to the main network, but with
 $ geth --testnet console
 ```
 
-The `console` subcommand have the exact same meaning as above and they are equally useful on the
-testnet too. Please see above for their explanations if you've skipped to here.
+The `console` subcommand has the exact same meaning as above and they are equally useful on the
+testnet too. Please see above for their explanations if you've skipped here.
 
-Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
+Specifying the `--testnet` flag, however, will reconfigure your Geth instance a bit:
 
  * Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
    itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
@@ -103,7 +103,7 @@ separate the two networks and will not make any accounts available between them.
 
 ### Full node on the Rinkeby test network
 
-The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
+The above test network is a cross-client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty/security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
 
 ```
 $ geth --rinkeby console
@@ -139,13 +139,13 @@ This will start geth in fast-sync mode with a DB memory allowance of 1GB just as
 
 Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
 
-### Programatically interfacing Geth nodes
+### Programmatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
 network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
-exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
+exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows).
 
 The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
 and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
@@ -168,12 +168,12 @@ HTTP based JSON-RPC API options:
   * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
 You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect
-via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification)
+via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification)
 on all transports. You can reuse the same connection for multiple requests!
 
 **Note: Please understand the security implications of opening up an HTTP/WS based transport before
 doing so! Hackers on the internet are actively trying to subvert Ethereum nodes with exposed APIs!
-Further, all browser tabs can access locally running webservers, so malicious webpages could try to
+Further, all browser tabs can access locally running web servers, so malicious web pages could try to
 subvert locally available APIs!**
 
 ### Operating a private network
@@ -241,7 +241,7 @@ that other nodes can use to connect to it and exchange peer information. Make su
 displayed IP address information (most probably `[::]`) with your externally accessible IP to get the
 actual `enode` URL.
 
-*Note: You could also use a full fledged Geth node as a bootnode, but it's the less recommended way.*
+*Note: You could also use a full-fledged Geth node as a bootnode, but it's the less recommended way.*
 
 #### Starting up your member nodes
 
@@ -264,7 +264,7 @@ an OpenCL or CUDA enabled `ethminer` instance. For information on such a setup,
 [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) and the [Genoil miner](https://github.com/Genoil/cpp-ethereum)
 repository.
 
-In a private network setting however, a single CPU miner instance is more than enough for practical
+In a private network setting, however a single CPU miner instance is more than enough for practical
 purposes as it can produce a stable stream of blocks at the correct intervals without needing heavy
 resources (consider running on a single thread, no need for multiple ones either). To start a Geth
 instance for mining, run it with all your usual flags, extended by:
@@ -298,7 +298,7 @@ Please make sure your contributions adhere to our coding guidelines:
    * E.g. "eth, rpc: make trace configs optional"
 
 Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, managing project dependencies and testing procedures.
+for more details on configuring your environment, managing project dependencies, and testing procedures.
 
 ## License
 

accounts/abi/abi.go

@@ -58,13 +58,11 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
 			return nil, err
 		}
 		return arguments, nil
-
 	}
 	method, exist := abi.Methods[name]
 	if !exist {
 		return nil, fmt.Errorf("method '%s' not found", name)
 	}
-
 	arguments, err := method.Inputs.Pack(args...)
 	if err != nil {
 		return nil, err
@@ -82,7 +80,7 @@ func (abi ABI) Unpack(v interface{}, name string, output []byte) (err error) {
 	// we need to decide whether we're calling a method or an event
 	if method, ok := abi.Methods[name]; ok {
 		if len(output)%32 != 0 {
-			return fmt.Errorf("abi: improperly formatted output")
+			return fmt.Errorf("abi: improperly formatted output: %s - Bytes: [%+v]", string(output), output)
 		}
 		return method.Outputs.Unpack(v, output)
 	} else if event, ok := abi.Events[name]; ok {
@@ -137,6 +135,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // MethodById looks up a method by the 4-byte id
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	if len(sigdata) < 4 {
+		return nil, fmt.Errorf("data too short (%d bytes) for abi method lookup", len(sigdata))
+	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {
 			return &method, nil

accounts/abi/abi_test.go

@@ -22,11 +22,10 @@ import (
 	"fmt"
 	"log"
 	"math/big"
+	"reflect"
 	"strings"
 	"testing"
 
-	"reflect"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
@@ -52,11 +51,14 @@ const jsondata2 = `
 	{ "type" : "function", "name" : "slice", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
 	{ "type" : "function", "name" : "slice256", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] },
 	{ "type" : "function", "name" : "sliceAddress", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] },
-	{ "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] }
+	{ "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] },
+	{ "type" : "function", "name" : "nestedArray", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] },
+	{ "type" : "function", "name" : "nestedArray2", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] },
+	{ "type" : "function", "name" : "nestedSlice", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] }
 ]`
 
 func TestReader(t *testing.T) {
-	Uint256, _ := NewType("uint256")
+	Uint256, _ := NewType("uint256", nil)
 	exp := ABI{
 		Methods: map[string]Method{
 			"balance": {
@@ -177,7 +179,7 @@ func TestTestSlice(t *testing.T) {
 }
 
 func TestMethodSignature(t *testing.T) {
-	String, _ := NewType("string")
+	String, _ := NewType("string", nil)
 	m := Method{"foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
 	exp := "foo(string,string)"
 	if m.Sig() != exp {
@@ -189,12 +191,31 @@ func TestMethodSignature(t *testing.T) {
 		t.Errorf("expected ids to match %x != %x", m.Id(), idexp)
 	}
 
-	uintt, _ := NewType("uint256")
+	uintt, _ := NewType("uint256", nil)
 	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
 	}
+
+	// Method with tuple arguments
+	s, _ := NewType("tuple", []ArgumentMarshaling{
+		{Name: "a", Type: "int256"},
+		{Name: "b", Type: "int256[]"},
+		{Name: "c", Type: "tuple[]", Components: []ArgumentMarshaling{
+			{Name: "x", Type: "int256"},
+			{Name: "y", Type: "int256"},
+		}},
+		{Name: "d", Type: "tuple[2]", Components: []ArgumentMarshaling{
+			{Name: "x", Type: "int256"},
+			{Name: "y", Type: "int256"},
+		}},
+	})
+	m = Method{"foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
+	exp = "foo((int256,int256[],(int256,int256)[],(int256,int256)[2]),string)"
+	if m.Sig() != exp {
+		t.Error("signature mismatch", exp, "!=", m.Sig())
+	}
 }
 
 func TestMultiPack(t *testing.T) {
@@ -564,11 +585,13 @@ func TestBareEvents(t *testing.T) {
 	const definition = `[
 	{ "type" : "event", "name" : "balance" },
 	{ "type" : "event", "name" : "anon", "anonymous" : true},
-	{ "type" : "event", "name" : "args", "inputs" : [{ "indexed":false, "name":"arg0", "type":"uint256" }, { "indexed":true, "name":"arg1", "type":"address" }] }
+	{ "type" : "event", "name" : "args", "inputs" : [{ "indexed":false, "name":"arg0", "type":"uint256" }, { "indexed":true, "name":"arg1", "type":"address" }] },
+	{ "type" : "event", "name" : "tuple", "inputs" : [{ "indexed":false, "name":"t", "type":"tuple", "components":[{"name":"a", "type":"uint256"}] }, { "indexed":true, "name":"arg1", "type":"address" }] }
 	]`
 
-	arg0, _ := NewType("uint256")
-	arg1, _ := NewType("address")
+	arg0, _ := NewType("uint256", nil)
+	arg1, _ := NewType("address", nil)
+	tuple, _ := NewType("tuple", []ArgumentMarshaling{{Name: "a", Type: "uint256"}})
 
 	expectedEvents := map[string]struct {
 		Anonymous bool
@@ -580,6 +603,10 @@ func TestBareEvents(t *testing.T) {
 			{Name: "arg0", Type: arg0, Indexed: false},
 			{Name: "arg1", Type: arg1, Indexed: true},
 		}},
+		"tuple": {false, []Argument{
+			{Name: "t", Type: tuple, Indexed: false},
+			{Name: "arg1", Type: arg1, Indexed: true},
+		}},
 	}
 
 	abi, err := JSON(strings.NewReader(definition))
@@ -646,28 +673,24 @@ func TestUnpackEvent(t *testing.T) {
 	}
 
 	type ReceivedEvent struct {
-		Address common.Address
-		Amount  *big.Int
-		Memo    []byte
+		Sender common.Address
+		Amount *big.Int
+		Memo   []byte
 	}
 	var ev ReceivedEvent
 
 	err = abi.Unpack(&ev, "received", data)
 	if err != nil {
 		t.Error(err)
-	} else {
-		t.Logf("len(data): %d; received event: %+v", len(data), ev)
 	}
 
 	type ReceivedAddrEvent struct {
-		Address common.Address
+		Sender common.Address
 	}
 	var receivedAddrEv ReceivedAddrEvent
 	err = abi.Unpack(&receivedAddrEv, "receivedAddr", data)
 	if err != nil {
 		t.Error(err)
-	} else {
-		t.Logf("len(data): %d; received event: %+v", len(data), receivedAddrEv)
 	}
 }
 
@@ -711,5 +734,14 @@ func TestABI_MethodById(t *testing.T) {
 			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
 		}
 	}
-
+	// Also test empty
+	if _, err := abi.MethodById([]byte{0x00}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById([]byte{}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById(nil); err == nil {
+		t.Errorf("Expected error, nil is short to decode data")
+	}
 }

accounts/abi/argument.go

@@ -33,24 +33,27 @@ type Argument struct {
 
 type Arguments []Argument
 
+type ArgumentMarshaling struct {
+	Name       string
+	Type       string
+	Components []ArgumentMarshaling
+	Indexed    bool
+}
+
 // UnmarshalJSON implements json.Unmarshaler interface
 func (argument *Argument) UnmarshalJSON(data []byte) error {
-	var extarg struct {
-		Name    string
-		Type    string
-		Indexed bool
-	}
-	err := json.Unmarshal(data, &extarg)
+	var arg ArgumentMarshaling
+	err := json.Unmarshal(data, &arg)
 	if err != nil {
 		return fmt.Errorf("argument json err: %v", err)
 	}
 
-	argument.Type, err = NewType(extarg.Type)
+	argument.Type, err = NewType(arg.Type, arg.Components)
 	if err != nil {
 		return err
 	}
-	argument.Name = extarg.Name
-	argument.Indexed = extarg.Indexed
+	argument.Name = arg.Name
+	argument.Indexed = arg.Indexed
 
 	return nil
 }
@@ -85,7 +88,6 @@ func (arguments Arguments) isTuple() bool {
 
 // Unpack performs the operation hexdata -> Go format
 func (arguments Arguments) Unpack(v interface{}, data []byte) error {
-
 	// make sure the passed value is arguments pointer
 	if reflect.Ptr != reflect.ValueOf(v).Kind() {
 		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
@@ -97,52 +99,134 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error {
 	if arguments.isTuple() {
 		return arguments.unpackTuple(v, marshalledValues)
 	}
-	return arguments.unpackAtomic(v, marshalledValues)
+	return arguments.unpackAtomic(v, marshalledValues[0])
 }
 
-func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error {
+// unpack sets the unmarshalled value to go format.
+// Note the dst here must be settable.
+func unpack(t *Type, dst interface{}, src interface{}) error {
+	var (
+		dstVal = reflect.ValueOf(dst).Elem()
+		srcVal = reflect.ValueOf(src)
+	)
 
+	if t.T != TupleTy && !((t.T == SliceTy || t.T == ArrayTy) && t.Elem.T == TupleTy) {
+		return set(dstVal, srcVal)
+	}
+
+	switch t.T {
+	case TupleTy:
+		if dstVal.Kind() != reflect.Struct {
+			return fmt.Errorf("abi: invalid dst value for unpack, want struct, got %s", dstVal.Kind())
+		}
+		fieldmap, err := mapArgNamesToStructFields(t.TupleRawNames, dstVal)
+		if err != nil {
+			return err
+		}
+		for i, elem := range t.TupleElems {
+			fname := fieldmap[t.TupleRawNames[i]]
+			field := dstVal.FieldByName(fname)
+			if !field.IsValid() {
+				return fmt.Errorf("abi: field %s can't found in the given value", t.TupleRawNames[i])
+			}
+			if err := unpack(elem, field.Addr().Interface(), srcVal.Field(i).Interface()); err != nil {
+				return err
+			}
+		}
+		return nil
+	case SliceTy:
+		if dstVal.Kind() != reflect.Slice {
+			return fmt.Errorf("abi: invalid dst value for unpack, want slice, got %s", dstVal.Kind())
+		}
+		slice := reflect.MakeSlice(dstVal.Type(), srcVal.Len(), srcVal.Len())
+		for i := 0; i < slice.Len(); i++ {
+			if err := unpack(t.Elem, slice.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil {
+				return err
+			}
+		}
+		dstVal.Set(slice)
+	case ArrayTy:
+		if dstVal.Kind() != reflect.Array {
+			return fmt.Errorf("abi: invalid dst value for unpack, want array, got %s", dstVal.Kind())
+		}
+		array := reflect.New(dstVal.Type()).Elem()
+		for i := 0; i < array.Len(); i++ {
+			if err := unpack(t.Elem, array.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil {
+				return err
+			}
+		}
+		dstVal.Set(array)
+	}
+	return nil
+}
+
+// unpackAtomic unpacks ( hexdata -> go ) a single value
+func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interface{}) error {
+	if arguments.LengthNonIndexed() == 0 {
+		return nil
+	}
+	argument := arguments.NonIndexed()[0]
+	elem := reflect.ValueOf(v).Elem()
+
+	if elem.Kind() == reflect.Struct {
+		fieldmap, err := mapArgNamesToStructFields([]string{argument.Name}, elem)
+		if err != nil {
+			return err
+		}
+		field := elem.FieldByName(fieldmap[argument.Name])
+		if !field.IsValid() {
+			return fmt.Errorf("abi: field %s can't be found in the given value", argument.Name)
+		}
+		return unpack(&argument.Type, field.Addr().Interface(), marshalledValues)
+	}
+	return unpack(&argument.Type, elem.Addr().Interface(), marshalledValues)
+}
+
+// unpackTuple unpacks ( hexdata -> go ) a batch of values.
+func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error {
 	var (
 		value = reflect.ValueOf(v).Elem()
 		typ   = value.Type()
 		kind  = value.Kind()
 	)
-
 	if err := requireUnpackKind(value, typ, kind, arguments); err != nil {
 		return err
 	}
 
 	// If the interface is a struct, get of abi->struct_field mapping
-
 	var abi2struct map[string]string
 	if kind == reflect.Struct {
-		var err error
-		abi2struct, err = mapAbiToStructFields(arguments, value)
+		var (
+			argNames []string
+			err      error
+		)
+		for _, arg := range arguments.NonIndexed() {
+			argNames = append(argNames, arg.Name)
+		}
+		abi2struct, err = mapArgNamesToStructFields(argNames, value)
 		if err != nil {
 			return err
 		}
 	}
 	for i, arg := range arguments.NonIndexed() {
-
-		reflectValue := reflect.ValueOf(marshalledValues[i])
-
 		switch kind {
 		case reflect.Struct:
-			if structField, ok := abi2struct[arg.Name]; ok {
-				if err := set(value.FieldByName(structField), reflectValue, arg); err != nil {
-					return err
-				}
+			field := value.FieldByName(abi2struct[arg.Name])
+			if !field.IsValid() {
+				return fmt.Errorf("abi: field %s can't be found in the given value", arg.Name)
+			}
+			if err := unpack(&arg.Type, field.Addr().Interface(), marshalledValues[i]); err != nil {
+				return err
 			}
 		case reflect.Slice, reflect.Array:
 			if value.Len() < i {
 				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len())
 			}
 			v := value.Index(i)
-			if err := requireAssignable(v, reflectValue); err != nil {
+			if err := requireAssignable(v, reflect.ValueOf(marshalledValues[i])); err != nil {
 				return err
 			}
-
-			if err := set(v.Elem(), reflectValue, arg); err != nil {
+			if err := unpack(&arg.Type, v.Addr().Interface(), marshalledValues[i]); err != nil {
 				return err
 			}
 		default:
@@ -150,48 +234,7 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa
 		}
 	}
 	return nil
-}
 
-// unpackAtomic unpacks ( hexdata -> go ) a single value
-func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error {
-	if len(marshalledValues) != 1 {
-		return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues))
-	}
-
-	elem := reflect.ValueOf(v).Elem()
-	kind := elem.Kind()
-	reflectValue := reflect.ValueOf(marshalledValues[0])
-
-	var abi2struct map[string]string
-	if kind == reflect.Struct {
-		var err error
-		if abi2struct, err = mapAbiToStructFields(arguments, elem); err != nil {
-			return err
-		}
-		arg := arguments.NonIndexed()[0]
-		if structField, ok := abi2struct[arg.Name]; ok {
-			return set(elem.FieldByName(structField), reflectValue, arg)
-		}
-		return nil
-	}
-
-	return set(elem, reflectValue, arguments.NonIndexed()[0])
-
-}
-
-// Computes the full size of an array;
-// i.e. counting nested arrays, which count towards size for unpacking.
-func getArraySize(arr *Type) int {
-	size := arr.Size
-	// Arrays can be nested, with each element being the same size
-	arr = arr.Elem
-	for arr.T == ArrayTy {
-		// Keep multiplying by elem.Size while the elem is an array.
-		size *= arr.Size
-		arr = arr.Elem
-	}
-	// Now we have the full array size, including its children.
-	return size
 }
 
 // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification,
@@ -202,7 +245,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
 	virtualArgs := 0
 	for index, arg := range arguments.NonIndexed() {
 		marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
-		if arg.Type.T == ArrayTy {
+		if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) {
 			// If we have a static array, like [3]uint256, these are coded as
 			// just like uint256,uint256,uint256.
 			// This means that we need to add two 'virtual' arguments when
@@ -213,7 +256,11 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
 			//
 			// Calculate the full array size to get the correct offset for the next argument.
 			// Decrement it by 1, as the normal index increment is still applied.
-			virtualArgs += getArraySize(&arg.Type) - 1
+			virtualArgs += getTypeSize(arg.Type)/32 - 1
+		} else if arg.Type.T == TupleTy && !isDynamicType(arg.Type) {
+			// If we have a static tuple, like (uint256, bool, uint256), these are
+			// coded as just like uint256,bool,uint256
+			virtualArgs += getTypeSize(arg.Type)/32 - 1
 		}
 		if err != nil {
 			return nil, err
@@ -243,11 +290,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 	// input offset is the bytes offset for packed output
 	inputOffset := 0
 	for _, abiArg := range abiArgs {
-		if abiArg.Type.T == ArrayTy {
-			inputOffset += 32 * abiArg.Type.Size
-		} else {
-			inputOffset += 32
-		}
+		inputOffset += getTypeSize(abiArg.Type)
 	}
 	var ret []byte
 	for i, a := range args {
@@ -257,14 +300,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-		// check for a slice type (string, bytes, slice)
-		if input.Type.requiresLengthPrefix() {
-			// calculate the offset
-			offset := inputOffset + len(variableInput)
+		// check for dynamic types
+		if isDynamicType(input.Type) {
 			// set the offset
-			ret = append(ret, packNum(reflect.ValueOf(offset))...)
-			// Append the packed output to the variable input. The variable input
-			// will be appended at the end of the input.
+			ret = append(ret, packNum(reflect.ValueOf(inputOffset))...)
+			// calculate next offset
+			inputOffset += len(packed)
+			// append to variable input
 			variableInput = append(variableInput, packed...)
 		} else {
 			// append the packed value to the input
@@ -277,14 +319,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 	return ret, nil
 }
 
-// capitalise makes the first character of a string upper case, also removing any
-// prefixing underscores from the variable names.
-func capitalise(input string) string {
-	for len(input) > 0 && input[0] == '_' {
-		input = input[1:]
+// ToCamelCase converts an under-score string to a camel-case string
+func ToCamelCase(input string) string {
+	parts := strings.Split(input, "_")
+	for i, s := range parts {
+		if len(s) > 0 {
+			parts[i] = strings.ToUpper(s[:1]) + s[1:]
+		}
 	}
-	if len(input) == 0 {
-		return ""
-	}
-	return strings.ToUpper(input[:1]) + input[1:]
+	return strings.Join(parts, "")
 }

accounts/abi/bind/backends/simulated.go

@@ -66,10 +66,10 @@ type SimulatedBackend struct {
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
 func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	database := ethdb.NewMemDatabase()
+	database := rawdb.NewMemoryDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
@@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 	return receipt, nil
 }
 
+// TransactionByHash checks the pool of pending transactions in addition to the
+// blockchain. The isPending return value indicates whether the transaction has been
+// mined yet. Note that the transaction may not be part of the canonical chain even if
+// it's not pending.
+func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (*types.Transaction, bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	tx := b.pendingBlock.Transaction(txHash)
+	if tx != nil {
+		return tx, true, nil
+	}
+	tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
+	if tx != nil {
+		return tx, false, nil
+	}
+	return nil, false, ethereum.NotFound
+}
+
 // PendingCodeAt returns the code associated with an account in the pending state.
 func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
 	b.mu.Lock()
@@ -208,7 +227,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
-// chain doens't have miners, we just return a gas price of 1 for any call.
+// chain doesn't have miners, we just return a gas price of 1 for any call.
 func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	return big.NewInt(1), nil
 }

accounts/abi/bind/backends/simulated_test.go

@@ -0,0 +1,82 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package backends_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+func TestSimulatedBackend(t *testing.T) {
+	var gasLimit uint64 = 8000029
+	key, _ := crypto.GenerateKey() // nolint: gosec
+	auth := bind.NewKeyedTransactor(key)
+	genAlloc := make(core.GenesisAlloc)
+	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
+
+	sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
+
+	// should return an error if the tx is not found
+	txHash := common.HexToHash("2")
+	_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
+
+	if isPending {
+		t.Fatal("transaction should not be pending")
+	}
+	if err != ethereum.NotFound {
+		t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
+	}
+
+	// generate a transaction and confirm you can retrieve it
+	code := `6060604052600a8060106000396000f360606040526008565b00`
+	var gas uint64 = 3000000
+	tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
+	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
+
+	err = sim.SendTransaction(context.Background(), tx)
+	if err != nil {
+		t.Fatal("error sending transaction")
+	}
+
+	txHash = tx.Hash()
+	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if !isPending {
+		t.Fatal("transaction should have pending status")
+	}
+
+	sim.Commit()
+	tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if isPending {
+		t.Fatal("transaction should not have pending status")
+	}
+
+}

accounts/abi/bind/base.go

@@ -36,10 +36,10 @@ type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Tra
 
 // CallOpts is the collection of options to fine tune a contract call request.
 type CallOpts struct {
-	Pending bool           // Whether to operate on the pending state or the last known one
-	From    common.Address // Optional the sender address, otherwise the first account is used
-
-	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+	Pending     bool            // Whether to operate on the pending state or the last known one
+	From        common.Address  // Optional the sender address, otherwise the first account is used
+	BlockNumber *big.Int        // Optional the block number on which the call should be performed
+	Context     context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
 
 // TransactOpts is the collection of authorization data required to create a
@@ -148,10 +148,10 @@ func (c *BoundContract) Call(opts *CallOpts, result interface{}, method string,
 			}
 		}
 	} else {
-		output, err = c.caller.CallContract(ctx, msg, nil)
+		output, err = c.caller.CallContract(ctx, msg, opts.BlockNumber)
 		if err == nil && len(output) == 0 {
 			// Make sure we have a contract to operate on, and bail out otherwise.
-			if code, err = c.caller.CodeAt(ctx, c.address, nil); err != nil {
+			if code, err = c.caller.CodeAt(ctx, c.address, opts.BlockNumber); err != nil {
 				return err
 			} else if len(code) == 0 {
 				return ErrNoCode

accounts/abi/bind/base_test.go

@@ -0,0 +1,80 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bind_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type mockCaller struct {
+	codeAtBlockNumber       *big.Int
+	callContractBlockNumber *big.Int
+}
+
+func (mc *mockCaller) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) {
+	mc.codeAtBlockNumber = blockNumber
+	return []byte{1, 2, 3}, nil
+}
+
+func (mc *mockCaller) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
+	mc.callContractBlockNumber = blockNumber
+	return nil, nil
+}
+
+func TestPassingBlockNumber(t *testing.T) {
+
+	mc := &mockCaller{}
+
+	bc := bind.NewBoundContract(common.HexToAddress("0x0"), abi.ABI{
+		Methods: map[string]abi.Method{
+			"something": {
+				Name:    "something",
+				Outputs: abi.Arguments{},
+			},
+		},
+	}, mc, nil, nil)
+	var ret string
+
+	blockNumber := big.NewInt(42)
+
+	bc.Call(&bind.CallOpts{BlockNumber: blockNumber}, &ret, "something")
+
+	if mc.callContractBlockNumber != blockNumber {
+		t.Fatalf("CallContract() was not passed the block number")
+	}
+
+	if mc.codeAtBlockNumber != blockNumber {
+		t.Fatalf("CodeAt() was not passed the block number")
+	}
+
+	bc.Call(&bind.CallOpts{}, &ret, "something")
+
+	if mc.callContractBlockNumber != nil {
+		t.Fatalf("CallContract() was passed a block number when it should not have been")
+	}
+
+	if mc.codeAtBlockNumber != nil {
+		t.Fatalf("CodeAt() was passed a block number when it should not have been")
+	}
+}

accounts/abi/bind/bind.go

@@ -23,13 +23,13 @@ package bind
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"regexp"
 	"strings"
 	"text/template"
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
-	"golang.org/x/tools/imports"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	if err := tmpl.Execute(buffer, data); err != nil {
 		return "", err
 	}
-	// For Go bindings pass the code through goimports to clean it up and double check
+	// For Go bindings pass the code through gofmt to clean it up
 	if lang == LangGo {
-		code, err := imports.Process(".", buffer.Bytes(), nil)
+		code, err := format.Source(buffer.Bytes())
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}
@@ -381,54 +381,23 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 // methodNormalizer is a name transformer that modifies Solidity method names to
 // conform to target language naming concentions.
 var methodNormalizer = map[Lang]func(string) string{
-	LangGo:   capitalise,
+	LangGo:   abi.ToCamelCase,
 	LangJava: decapitalise,
 }
 
 // capitalise makes a camel-case string which starts with an upper case character.
 func capitalise(input string) string {
-	for len(input) > 0 && input[0] == '_' {
-		input = input[1:]
-	}
-	if len(input) == 0 {
-		return ""
-	}
-	return toCamelCase(strings.ToUpper(input[:1]) + input[1:])
+	return abi.ToCamelCase(input)
 }
 
 // decapitalise makes a camel-case string which starts with a lower case character.
 func decapitalise(input string) string {
-	for len(input) > 0 && input[0] == '_' {
-		input = input[1:]
-	}
 	if len(input) == 0 {
-		return ""
+		return input
 	}
-	return toCamelCase(strings.ToLower(input[:1]) + input[1:])
-}
 
-// toCamelCase converts an under-score string to a camel-case string
-func toCamelCase(input string) string {
-	toupper := false
-
-	result := ""
-	for k, v := range input {
-		switch {
-		case k == 0:
-			result = strings.ToUpper(string(input[0]))
-
-		case toupper:
-			result += strings.ToUpper(string(v))
-			toupper = false
-
-		case v == '_':
-			toupper = true
-
-		default:
-			result += string(v)
-		}
-	}
-	return result
+	goForm := abi.ToCamelCase(input)
+	return strings.ToLower(goForm[:1]) + goForm[1:]
 }
 
 // structured checks whether a list of ABI data types has enough information to

accounts/abi/bind/template.go

@@ -64,6 +64,30 @@ const tmplSourceGo = `
 
 package {{.Package}}
 
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
 {{range $contract := .Contracts}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"

accounts/abi/bind/topics.go

@@ -83,8 +83,10 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
 				val := reflect.ValueOf(rule)
 
 				switch {
+
+				// static byte array
 				case val.Kind() == reflect.Array && reflect.TypeOf(rule).Elem().Kind() == reflect.Uint8:
-					reflect.Copy(reflect.ValueOf(topic[common.HashLength-val.Len():]), val)
+					reflect.Copy(reflect.ValueOf(topic[:val.Len()]), val)
 
 				default:
 					return nil, fmt.Errorf("unsupported indexed type: %T", rule)
@@ -175,8 +177,10 @@ func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) er
 			default:
 				// Ran out of custom types, try the crazies
 				switch {
+
+				// static byte array
 				case arg.Type.T == abi.FixedBytesTy:
-					reflect.Copy(field, reflect.ValueOf(topics[0][common.HashLength-arg.Type.Size:]))
+					reflect.Copy(field, reflect.ValueOf(topics[0][:arg.Type.Size]))
 
 				default:
 					return fmt.Errorf("unsupported indexed type: %v", arg.Type)

accounts/abi/bind/topics_test.go

@@ -0,0 +1,103 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bind
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func TestMakeTopics(t *testing.T) {
+	type args struct {
+		query [][]interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    [][]common.Hash
+		wantErr bool
+	}{
+		{
+			"support fixed byte types, right padded to 32 bytes",
+			args{[][]interface{}{{[5]byte{1, 2, 3, 4, 5}}}},
+			[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := makeTopics(tt.args.query...)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("makeTopics() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("makeTopics() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseTopics(t *testing.T) {
+	type bytesStruct struct {
+		StaticBytes [5]byte
+	}
+	bytesType, _ := abi.NewType("bytes5", nil)
+	type args struct {
+		createObj func() interface{}
+		resultObj func() interface{}
+		fields    abi.Arguments
+		topics    []common.Hash
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "support fixed byte types, right padded to 32 bytes",
+			args: args{
+				createObj: func() interface{} { return &bytesStruct{} },
+				resultObj: func() interface{} { return &bytesStruct{StaticBytes: [5]byte{1, 2, 3, 4, 5}} },
+				fields: abi.Arguments{abi.Argument{
+					Name:    "staticBytes",
+					Type:    bytesType,
+					Indexed: true,
+				}},
+				topics: []common.Hash{
+					{1, 2, 3, 4, 5},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			createObj := tt.args.createObj()
+			if err := parseTopics(createObj, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr {
+				t.Errorf("parseTopics() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			resultObj := tt.args.resultObj()
+			if !reflect.DeepEqual(createObj, resultObj) {
+				t.Errorf("parseTopics() = %v, want %v", createObj, resultObj)
+			}
+		})
+	}
+}

accounts/abi/event.go

@@ -36,12 +36,12 @@ type Event struct {
 func (e Event) String() string {
 	inputs := make([]string, len(e.Inputs))
 	for i, input := range e.Inputs {
-		inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
+		inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name)
 		if input.Indexed {
-			inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type)
+			inputs[i] = fmt.Sprintf("%v indexed %v", input.Type, input.Name)
 		}
 	}
-	return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", "))
+	return fmt.Sprintf("event %v(%v)", e.Name, strings.Join(inputs, ", "))
 }
 
 // Id returns the canonical representation of the event's signature used by the

accounts/abi/event_test.go

@@ -87,12 +87,12 @@ func TestEventId(t *testing.T) {
 	}{
 		{
 			definition: `[
-			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
-			{ "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }
+			{ "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
+			{ "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }
 			]`,
 			expectations: map[string]common.Hash{
-				"balance": crypto.Keccak256Hash([]byte("balance(uint256)")),
-				"check":   crypto.Keccak256Hash([]byte("check(address,uint256)")),
+				"Balance": crypto.Keccak256Hash([]byte("Balance(uint256)")),
+				"Check":   crypto.Keccak256Hash([]byte("Check(address,uint256)")),
 			},
 		},
 	}
@@ -111,6 +111,39 @@ func TestEventId(t *testing.T) {
 	}
 }
 
+func TestEventString(t *testing.T) {
+	var table = []struct {
+		definition   string
+		expectations map[string]string
+	}{
+		{
+			definition: `[
+			{ "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
+			{ "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] },
+			{ "type" : "event", "name" : "Transfer", "inputs": [{ "name": "from", "type": "address", "indexed": true }, { "name": "to", "type": "address", "indexed": true }, { "name": "value", "type": "uint256" }] }
+			]`,
+			expectations: map[string]string{
+				"Balance":  "event Balance(uint256 in)",
+				"Check":    "event Check(address t, uint256 b)",
+				"Transfer": "event Transfer(address indexed from, address indexed to, uint256 value)",
+			},
+		},
+	}
+
+	for _, test := range table {
+		abi, err := JSON(strings.NewReader(test.definition))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for name, event := range abi.Events {
+			if event.String() != test.expectations[name] {
+				t.Errorf("expected string to be %s, got %s", test.expectations[name], event.String())
+			}
+		}
+	}
+}
+
 // TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array.
 func TestEventMultiValueWithArrayUnpack(t *testing.T) {
 	definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]`

accounts/abi/method.go

@@ -56,14 +56,14 @@ func (method Method) Sig() string {
 func (method Method) String() string {
 	inputs := make([]string, len(method.Inputs))
 	for i, input := range method.Inputs {
-		inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
+		inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name)
 	}
 	outputs := make([]string, len(method.Outputs))
 	for i, output := range method.Outputs {
+		outputs[i] = output.Type.String()
 		if len(output.Name) > 0 {
-			outputs[i] = fmt.Sprintf("%v ", output.Name)
+			outputs[i] += fmt.Sprintf(" %v", output.Name)
 		}
-		outputs[i] += output.Type.String()
 	}
 	constant := ""
 	if method.Const {

accounts/abi/method_test.go

@@ -0,0 +1,61 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package abi
+
+import (
+	"strings"
+	"testing"
+)
+
+const methoddata = `
+[
+	{ "type" : "function", "name" : "balance", "constant" : true },
+	{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
+	{ "type" : "function", "name" : "transfer", "constant" : false, "inputs" : [ { "name" : "from", "type" : "address" }, { "name" : "to", "type" : "address" }, { "name" : "value", "type" : "uint256" } ], "outputs" : [ { "name" : "success", "type" : "bool" } ]  }
+]`
+
+func TestMethodString(t *testing.T) {
+	var table = []struct {
+		method      string
+		expectation string
+	}{
+		{
+			method:      "balance",
+			expectation: "function balance() constant returns()",
+		},
+		{
+			method:      "send",
+			expectation: "function send(uint256 amount) returns()",
+		},
+		{
+			method:      "transfer",
+			expectation: "function transfer(address from, address to, uint256 value) returns(bool success)",
+		},
+	}
+
+	abi, err := JSON(strings.NewReader(methoddata))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range table {
+		got := abi.Methods[test.method].String()
+		if got != test.expectation {
+			t.Errorf("expected string to be %s, got %s", test.expectation, got)
+		}
+	}
+}

accounts/abi/pack_test.go

@@ -29,314 +29,601 @@ import (
 
 func TestPack(t *testing.T) {
 	for i, test := range []struct {
-		typ string
-
-		input  interface{}
-		output []byte
+		typ        string
+		components []ArgumentMarshaling
+		input      interface{}
+		output     []byte
 	}{
 		{
 			"uint8",
+			nil,
 			uint8(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint8[]",
+			nil,
 			[]uint8{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint16",
+			nil,
 			uint16(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint16[]",
+			nil,
 			[]uint16{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint32",
+			nil,
 			uint32(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint32[]",
+			nil,
 			[]uint32{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint64",
+			nil,
 			uint64(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint64[]",
+			nil,
 			[]uint64{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint256",
+			nil,
 			big.NewInt(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"uint256[]",
+			nil,
 			[]*big.Int{big.NewInt(1), big.NewInt(2)},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int8",
+			nil,
 			int8(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int8[]",
+			nil,
 			[]int8{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int16",
+			nil,
 			int16(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int16[]",
+			nil,
 			[]int16{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int32",
+			nil,
 			int32(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int32[]",
+			nil,
 			[]int32{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int64",
+			nil,
 			int64(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int64[]",
+			nil,
 			[]int64{1, 2},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int256",
+			nil,
 			big.NewInt(2),
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"int256[]",
+			nil,
 			[]*big.Int{big.NewInt(1), big.NewInt(2)},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"),
 		},
 		{
 			"bytes1",
+			nil,
 			[1]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes2",
+			nil,
 			[2]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes3",
+			nil,
 			[3]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes4",
+			nil,
 			[4]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes5",
+			nil,
 			[5]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes6",
+			nil,
 			[6]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes7",
+			nil,
 			[7]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes8",
+			nil,
 			[8]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes9",
+			nil,
 			[9]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes10",
+			nil,
 			[10]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes11",
+			nil,
 			[11]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes12",
+			nil,
 			[12]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes13",
+			nil,
 			[13]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes14",
+			nil,
 			[14]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes15",
+			nil,
 			[15]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes16",
+			nil,
 			[16]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes17",
+			nil,
 			[17]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes18",
+			nil,
 			[18]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes19",
+			nil,
 			[19]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes20",
+			nil,
 			[20]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes21",
+			nil,
 			[21]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes22",
+			nil,
 			[22]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes23",
+			nil,
 			[23]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes24",
-			[24]byte{1},
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
-		},
-		{
-			"bytes24",
+			nil,
 			[24]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes25",
+			nil,
 			[25]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes26",
+			nil,
 			[26]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes27",
+			nil,
 			[27]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes28",
+			nil,
 			[28]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes29",
+			nil,
 			[29]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes30",
+			nil,
 			[30]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes31",
+			nil,
 			[31]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"bytes32",
+			nil,
 			[32]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"uint32[2][3][4]",
+			nil,
 			[4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018"),
 		},
 		{
 			"address[]",
+			nil,
 			[]common.Address{{1}, {2}},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000"),
 		},
 		{
 			"bytes32[]",
+			nil,
 			[]common.Hash{{1}, {2}},
 			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000201000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"function",
+			nil,
 			[24]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
 		{
 			"string",
+			nil,
 			"foobar",
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"string[]",
+			nil,
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"string[2]",
+			nil,
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"bytes32[][]",
+			nil,
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[][2]",
+			nil,
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[3][2]",
+			nil,
+			[][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+		{
+			// static tuple
+			"tuple",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "int64"},
+				{Name: "b", Type: "int256"},
+				{Name: "c", Type: "int256"},
+				{Name: "d", Type: "bool"},
+				{Name: "e", Type: "bytes32[3][2]"},
+			},
+			struct {
+				A int64
+				B *big.Int
+				C *big.Int
+				D bool
+				E [][]common.Hash
+			}{1, big.NewInt(1), big.NewInt(-1), true, [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001" + // struct[a]
+				"0000000000000000000000000000000000000000000000000000000000000001" + // struct[b]
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // struct[c]
+				"0000000000000000000000000000000000000000000000000000000000000001" + // struct[d]
+				"0100000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][1]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][2]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // struct[e] array[1][2]
+		},
+		{
+			// dynamic tuple
+			"tuple",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "string"},
+				{Name: "b", Type: "int64"},
+				{Name: "c", Type: "bytes"},
+				{Name: "d", Type: "string[]"},
+				{Name: "e", Type: "int256[]"},
+				{Name: "f", Type: "address[]"},
+			},
+			struct {
+				FieldA string `abi:"a"` // Test whether abi tag works
+				FieldB int64  `abi:"b"`
+				C      []byte
+				D      []string
+				E      []*big.Int
+				F      []common.Address
+			}{"foobar", 1, []byte{1}, []string{"foo", "bar"}, []*big.Int{big.NewInt(1), big.NewInt(-1)}, []common.Address{{1}, {2}}},
+			common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000c0" + // struct[a] offset
+				"0000000000000000000000000000000000000000000000000000000000000001" + // struct[b]
+				"0000000000000000000000000000000000000000000000000000000000000100" + // struct[c] offset
+				"0000000000000000000000000000000000000000000000000000000000000140" + // struct[d] offset
+				"0000000000000000000000000000000000000000000000000000000000000220" + // struct[e] offset
+				"0000000000000000000000000000000000000000000000000000000000000280" + // struct[f] offset
+				"0000000000000000000000000000000000000000000000000000000000000006" + // struct[a] length
+				"666f6f6261720000000000000000000000000000000000000000000000000000" + // struct[a] "foobar"
+				"0000000000000000000000000000000000000000000000000000000000000001" + // struct[c] length
+				"0100000000000000000000000000000000000000000000000000000000000000" + // []byte{1}
+				"0000000000000000000000000000000000000000000000000000000000000002" + // struct[d] length
+				"0000000000000000000000000000000000000000000000000000000000000040" + // foo offset
+				"0000000000000000000000000000000000000000000000000000000000000080" + // bar offset
+				"0000000000000000000000000000000000000000000000000000000000000003" + // foo length
+				"666f6f0000000000000000000000000000000000000000000000000000000000" + // foo
+				"0000000000000000000000000000000000000000000000000000000000000003" + // bar offset
+				"6261720000000000000000000000000000000000000000000000000000000000" + // bar
+				"0000000000000000000000000000000000000000000000000000000000000002" + // struct[e] length
+				"0000000000000000000000000000000000000000000000000000000000000001" + // 1
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // -1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // struct[f] length
+				"0000000000000000000000000100000000000000000000000000000000000000" + // common.Address{1}
+				"0000000000000000000000000200000000000000000000000000000000000000"), // common.Address{2}
+		},
+		{
+			// nested tuple
+			"tuple",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "tuple", Components: []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256[]"}}},
+				{Name: "b", Type: "int256[]"},
+			},
+			struct {
+				A struct {
+					FieldA *big.Int `abi:"a"`
+					B      []*big.Int
+				}
+				B []*big.Int
+			}{
+				A: struct {
+					FieldA *big.Int `abi:"a"` // Test whether abi tag works for nested tuple
+					B      []*big.Int
+				}{big.NewInt(1), []*big.Int{big.NewInt(1), big.NewInt(0)}},
+				B: []*big.Int{big.NewInt(1), big.NewInt(0)}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // a offset
+				"00000000000000000000000000000000000000000000000000000000000000e0" + // b offset
+				"0000000000000000000000000000000000000000000000000000000000000001" + // a.a value
+				"0000000000000000000000000000000000000000000000000000000000000040" + // a.b offset
+				"0000000000000000000000000000000000000000000000000000000000000002" + // a.b length
+				"0000000000000000000000000000000000000000000000000000000000000001" + // a.b[0] value
+				"0000000000000000000000000000000000000000000000000000000000000000" + // a.b[1] value
+				"0000000000000000000000000000000000000000000000000000000000000002" + // b length
+				"0000000000000000000000000000000000000000000000000000000000000001" + // b[0] value
+				"0000000000000000000000000000000000000000000000000000000000000000"), // b[1] value
+		},
+		{
+			// tuple slice
+			"tuple[]",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "int256"},
+				{Name: "b", Type: "int256[]"},
+			},
+			[]struct {
+				A *big.Int
+				B []*big.Int
+			}{
+				{big.NewInt(-1), []*big.Int{big.NewInt(1), big.NewInt(0)}},
+				{big.NewInt(1), []*big.Int{big.NewInt(2), big.NewInt(-1)}},
+			},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // tuple length
+				"0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0] offset
+				"00000000000000000000000000000000000000000000000000000000000000e0" + // tuple[1] offset
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].A
+				"0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0].B offset
+				"0000000000000000000000000000000000000000000000000000000000000002" + // tuple[0].B length
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].B[0] value
+				"0000000000000000000000000000000000000000000000000000000000000000" + // tuple[0].B[1] value
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].A
+				"0000000000000000000000000000000000000000000000000000000000000040" + // tuple[1].B offset
+				"0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].B length
+				"0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].B[0] value
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].B[1] value
+		},
+		{
+			// static tuple array
+			"tuple[2]",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "int256"},
+				{Name: "b", Type: "int256"},
+			},
+			[2]struct {
+				A *big.Int
+				B *big.Int
+			}{
+				{big.NewInt(-1), big.NewInt(1)},
+				{big.NewInt(1), big.NewInt(-1)},
+			},
+			common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].a
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].b
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].a
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].b
+		},
+		{
+			// dynamic tuple array
+			"tuple[2]",
+			[]ArgumentMarshaling{
+				{Name: "a", Type: "int256[]"},
+			},
+			[2]struct {
+				A []*big.Int
+			}{
+				{[]*big.Int{big.NewInt(-1), big.NewInt(1)}},
+				{[]*big.Int{big.NewInt(1), big.NewInt(-1)}},
+			},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0] offset
+				"00000000000000000000000000000000000000000000000000000000000000c0" + // tuple[1] offset
+				"0000000000000000000000000000000000000000000000000000000000000020" + // tuple[0].A offset
+				"0000000000000000000000000000000000000000000000000000000000000002" + // tuple[0].A length
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].A[0]
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].A[1]
+				"0000000000000000000000000000000000000000000000000000000000000020" + // tuple[1].A offset
+				"0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].A length
+				"0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].A[0]
+				"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].A[1]
+		},
 	} {
-		typ, err := NewType(test.typ)
+		typ, err := NewType(test.typ, test.components)
 		if err != nil {
 			t.Fatalf("%v failed. Unexpected parse error: %v", i, err)
 		}
-
 		output, err := typ.pack(reflect.ValueOf(test.input))
 		if err != nil {
 			t.Fatalf("%v failed. Unexpected pack error: %v", i, err)
 		}
 
 		if !bytes.Equal(output, test.output) {
-			t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output)
+			t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output)
 		}
 	}
 }
@@ -406,6 +693,59 @@ func TestMethodPack(t *testing.T) {
 	if !bytes.Equal(packed, sig) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
+
+	a := [2][2]*big.Int{{big.NewInt(1), big.NewInt(1)}, {big.NewInt(2), big.NewInt(0)}}
+	sig = abi.Methods["nestedArray"].Id()
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0xa0}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	sig = append(sig, common.LeftPadBytes(addrC[:], 32)...)
+	sig = append(sig, common.LeftPadBytes(addrD[:], 32)...)
+	packed, err = abi.Pack("nestedArray", a, []common.Address{addrC, addrD})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(packed, sig) {
+		t.Errorf("expected %x got %x", sig, packed)
+	}
+
+	sig = abi.Methods["nestedArray2"].Id()
+	sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0x80}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	packed, err = abi.Pack("nestedArray2", [2][]uint8{{1}, {1}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(packed, sig) {
+		t.Errorf("expected %x got %x", sig, packed)
+	}
+
+	sig = abi.Methods["nestedSlice"].Id()
+	sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0x02}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{0xa0}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
+	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
+	packed, err = abi.Pack("nestedSlice", [][]uint8{{1, 2}, {1, 2}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(packed, sig) {
+		t.Errorf("expected %x got %x", sig, packed)
+	}
 }
 
 func TestPackNumber(t *testing.T) {

accounts/abi/reflect.go

@@ -71,22 +71,36 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 //
 // set is a bit more lenient when it comes to assignment and doesn't force an as
 // strict ruleset as bare `reflect` does.
-func set(dst, src reflect.Value, output Argument) error {
-	dstType := dst.Type()
-	srcType := src.Type()
+func set(dst, src reflect.Value) error {
+	dstType, srcType := dst.Type(), src.Type()
 	switch {
-	case dstType.AssignableTo(srcType):
+	case dstType.Kind() == reflect.Interface && dst.Elem().IsValid():
+		return set(dst.Elem(), src)
+	case dstType.Kind() == reflect.Ptr && dstType.Elem() != derefbigT:
+		return set(dst.Elem(), src)
+	case srcType.AssignableTo(dstType) && dst.CanSet():
 		dst.Set(src)
-	case dstType.Kind() == reflect.Interface:
-		dst.Set(src)
-	case dstType.Kind() == reflect.Ptr:
-		return set(dst.Elem(), src, output)
+	case dstType.Kind() == reflect.Slice && srcType.Kind() == reflect.Slice:
+		return setSlice(dst, src)
 	default:
 		return fmt.Errorf("abi: cannot unmarshal %v in to %v", src.Type(), dst.Type())
 	}
 	return nil
 }
 
+// setSlice attempts to assign src to dst when slices are not assignable by default
+// e.g. src: [][]byte -> dst: [][15]byte
+func setSlice(dst, src reflect.Value) error {
+	slice := reflect.MakeSlice(dst.Type(), src.Len(), src.Len())
+	for i := 0; i < src.Len(); i++ {
+		v := src.Index(i)
+		reflect.Copy(slice.Index(i), v)
+	}
+
+	dst.Set(slice)
+	return nil
+}
+
 // requireAssignable assures that `dest` is a pointer and it's not an interface.
 func requireAssignable(dst, src reflect.Value) error {
 	if dst.Kind() != reflect.Ptr && dst.Kind() != reflect.Interface {
@@ -112,14 +126,14 @@ func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind,
 	return nil
 }
 
-// mapAbiToStringField maps abi to struct fields.
+// mapArgNamesToStructFields maps a slice of argument names to struct fields.
 // first round: for each Exportable field that contains a `abi:""` tag
-//   and this field name exists in the arguments, pair them together.
-// second round: for each argument field that has not been already linked,
+//   and this field name exists in the given argument name list, pair them together.
+// second round: for each argument name that has not been already linked,
 //   find what variable is expected to be mapped into, if it exists and has not been
 //   used, pair them.
-func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]string, error) {
-
+// Note this function assumes the given value is a struct value.
+func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[string]string, error) {
 	typ := value.Type()
 
 	abi2struct := make(map[string]string)
@@ -133,45 +147,39 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin
 		if structFieldName[:1] != strings.ToUpper(structFieldName[:1]) {
 			continue
 		}
-
 		// skip fields that have no abi:"" tag.
 		var ok bool
 		var tagName string
 		if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok {
 			continue
 		}
-
 		// check if tag is empty.
 		if tagName == "" {
 			return nil, fmt.Errorf("struct: abi tag in '%s' is empty", structFieldName)
 		}
-
 		// check which argument field matches with the abi tag.
 		found := false
-		for _, abiField := range args.NonIndexed() {
-			if abiField.Name == tagName {
-				if abi2struct[abiField.Name] != "" {
+		for _, arg := range argNames {
+			if arg == tagName {
+				if abi2struct[arg] != "" {
 					return nil, fmt.Errorf("struct: abi tag in '%s' already mapped", structFieldName)
 				}
 				// pair them
-				abi2struct[abiField.Name] = structFieldName
-				struct2abi[structFieldName] = abiField.Name
+				abi2struct[arg] = structFieldName
+				struct2abi[structFieldName] = arg
 				found = true
 			}
 		}
-
 		// check if this tag has been mapped.
 		if !found {
 			return nil, fmt.Errorf("struct: abi tag '%s' defined but not found in abi", tagName)
 		}
-
 	}
 
 	// second round ~~~
-	for _, arg := range args {
+	for _, argName := range argNames {
 
-		abiFieldName := arg.Name
-		structFieldName := capitalise(abiFieldName)
+		structFieldName := ToCamelCase(argName)
 
 		if structFieldName == "" {
 			return nil, fmt.Errorf("abi: purely underscored output cannot unpack to struct")
@@ -181,11 +189,11 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin
 		// struct field with the same field name. If so, raise an error:
 		//    abi: [ { "name": "value" } ]
 		//    struct { Value  *big.Int , Value1 *big.Int `abi:"value"`}
-		if abi2struct[abiFieldName] != "" {
-			if abi2struct[abiFieldName] != structFieldName &&
+		if abi2struct[argName] != "" {
+			if abi2struct[argName] != structFieldName &&
 				struct2abi[structFieldName] == "" &&
 				value.FieldByName(structFieldName).IsValid() {
-				return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", abiFieldName)
+				return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", argName)
 			}
 			continue
 		}
@@ -197,16 +205,14 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin
 
 		if value.FieldByName(structFieldName).IsValid() {
 			// pair them
-			abi2struct[abiFieldName] = structFieldName
-			struct2abi[structFieldName] = abiFieldName
+			abi2struct[argName] = structFieldName
+			struct2abi[structFieldName] = argName
 		} else {
 			// not paired, but annotate as used, to detect cases like
 			//   abi : [ { "name": "value" }, { "name": "_value" } ]
 			//   struct { Value *big.Int }
-			struct2abi[structFieldName] = abiFieldName
+			struct2abi[structFieldName] = argName
 		}
-
 	}
-
 	return abi2struct, nil
 }

accounts/abi/reflect_test.go

@@ -0,0 +1,191 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package abi
+
+import (
+	"reflect"
+	"testing"
+)
+
+type reflectTest struct {
+	name  string
+	args  []string
+	struc interface{}
+	want  map[string]string
+	err   string
+}
+
+var reflectTests = []reflectTest{
+	{
+		name: "OneToOneCorrespondance",
+		args: []string{"fieldA"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+		},
+	},
+	{
+		name: "MissingFieldsInStruct",
+		args: []string{"fieldA", "fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+		},
+	},
+	{
+		name: "MoreFieldsInStructThanArgs",
+		args: []string{"fieldA"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+			FieldB int
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+		},
+	},
+	{
+		name: "MissingFieldInArgs",
+		args: []string{"fieldA"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+			FieldB int `abi:"fieldB"`
+		}{},
+		err: "struct: abi tag 'fieldB' defined but not found in abi",
+	},
+	{
+		name: "NoAbiDescriptor",
+		args: []string{"fieldA"},
+		struc: struct {
+			FieldA int
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+		},
+	},
+	{
+		name: "NoArgs",
+		args: []string{},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+		}{},
+		err: "struct: abi tag 'fieldA' defined but not found in abi",
+	},
+	{
+		name: "DifferentName",
+		args: []string{"fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldB"`
+		}{},
+		want: map[string]string{
+			"fieldB": "FieldA",
+		},
+	},
+	{
+		name: "DifferentName",
+		args: []string{"fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldB"`
+		}{},
+		want: map[string]string{
+			"fieldB": "FieldA",
+		},
+	},
+	{
+		name: "MultipleFields",
+		args: []string{"fieldA", "fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+			FieldB int `abi:"fieldB"`
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+			"fieldB": "FieldB",
+		},
+	},
+	{
+		name: "MultipleFieldsABIMissing",
+		args: []string{"fieldA", "fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldA"`
+			FieldB int
+		}{},
+		want: map[string]string{
+			"fieldA": "FieldA",
+			"fieldB": "FieldB",
+		},
+	},
+	{
+		name: "NameConflict",
+		args: []string{"fieldB"},
+		struc: struct {
+			FieldA int `abi:"fieldB"`
+			FieldB int
+		}{},
+		err: "abi: multiple variables maps to the same abi field 'fieldB'",
+	},
+	{
+		name: "Underscored",
+		args: []string{"_"},
+		struc: struct {
+			FieldA int
+		}{},
+		err: "abi: purely underscored output cannot unpack to struct",
+	},
+	{
+		name: "DoubleMapping",
+		args: []string{"fieldB", "fieldC", "fieldA"},
+		struc: struct {
+			FieldA int `abi:"fieldC"`
+			FieldB int
+		}{},
+		err: "abi: multiple outputs mapping to the same struct field 'FieldA'",
+	},
+	{
+		name: "AlreadyMapped",
+		args: []string{"fieldB", "fieldB"},
+		struc: struct {
+			FieldB int `abi:"fieldB"`
+		}{},
+		err: "struct: abi tag in 'FieldB' already mapped",
+	},
+}
+
+func TestReflectNameToStruct(t *testing.T) {
+	for _, test := range reflectTests {
+		t.Run(test.name, func(t *testing.T) {
+			m, err := mapArgNamesToStructFields(test.args, reflect.ValueOf(test.struc))
+			if len(test.err) > 0 {
+				if err == nil || err.Error() != test.err {
+					t.Fatalf("Invalid error: expected %v, got %v", test.err, err)
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("Unexpected error: %v", err)
+				}
+				for fname := range test.want {
+					if m[fname] != test.want[fname] {
+						t.Fatalf("Incorrect value for field %s: expected %v, got %v", fname, test.want[fname], m[fname])
+					}
+				}
+			}
+		})
+	}
+}

accounts/abi/type.go

@@ -17,6 +17,7 @@
 package abi
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"regexp"
@@ -32,6 +33,7 @@ const (
 	StringTy
 	SliceTy
 	ArrayTy
+	TupleTy
 	AddressTy
 	FixedBytesTy
 	BytesTy
@@ -43,13 +45,16 @@ const (
 // Type is the reflection of the supported argument type
 type Type struct {
 	Elem *Type
-
 	Kind reflect.Kind
 	Type reflect.Type
 	Size int
 	T    byte // Our own type checking
 
 	stringKind string // holds the unparsed string for deriving signatures
+
+	// Tuple relative fields
+	TupleElems    []*Type  // Type information of all tuple fields
+	TupleRawNames []string // Raw field name of all tuple fields
 }
 
 var (
@@ -58,7 +63,7 @@ var (
 )
 
 // NewType creates a new reflection type of abi type given in t.
-func NewType(t string) (typ Type, err error) {
+func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) {
 	// check that array brackets are equal if they exist
 	if strings.Count(t, "[") != strings.Count(t, "]") {
 		return Type{}, fmt.Errorf("invalid arg type in abi")
@@ -71,7 +76,7 @@ func NewType(t string) (typ Type, err error) {
 	if strings.Count(t, "[") != 0 {
 		i := strings.LastIndex(t, "[")
 		// recursively embed the type
-		embeddedType, err := NewType(t[:i])
+		embeddedType, err := NewType(t[:i], components)
 		if err != nil {
 			return Type{}, err
 		}
@@ -87,6 +92,9 @@ func NewType(t string) (typ Type, err error) {
 			typ.Kind = reflect.Slice
 			typ.Elem = &embeddedType
 			typ.Type = reflect.SliceOf(embeddedType.Type)
+			if embeddedType.T == TupleTy {
+				typ.stringKind = embeddedType.stringKind + sliced
+			}
 		} else if len(intz) == 1 {
 			// is a array
 			typ.T = ArrayTy
@@ -97,13 +105,21 @@ func NewType(t string) (typ Type, err error) {
 				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
 			}
 			typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type)
+			if embeddedType.T == TupleTy {
+				typ.stringKind = embeddedType.stringKind + sliced
+			}
 		} else {
 			return Type{}, fmt.Errorf("invalid formatting of array type")
 		}
 		return typ, err
 	}
 	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+	matches := typeRegex.FindAllStringSubmatch(t, -1)
+	if len(matches) == 0 {
+		return Type{}, fmt.Errorf("invalid type '%v'", t)
+	}
+	parsedType := matches[0]
+
 	// varSize is the size of the variable
 	var varSize int
 	if len(parsedType[3]) > 0 {
@@ -153,6 +169,40 @@ func NewType(t string) (typ Type, err error) {
 			typ.Size = varSize
 			typ.Type = reflect.ArrayOf(varSize, reflect.TypeOf(byte(0)))
 		}
+	case "tuple":
+		var (
+			fields     []reflect.StructField
+			elems      []*Type
+			names      []string
+			expression string // canonical parameter expression
+		)
+		expression += "("
+		for idx, c := range components {
+			cType, err := NewType(c.Type, c.Components)
+			if err != nil {
+				return Type{}, err
+			}
+			if ToCamelCase(c.Name) == "" {
+				return Type{}, errors.New("abi: purely anonymous or underscored field is not supported")
+			}
+			fields = append(fields, reflect.StructField{
+				Name: ToCamelCase(c.Name), // reflect.StructOf will panic for any exported field.
+				Type: cType.Type,
+			})
+			elems = append(elems, &cType)
+			names = append(names, c.Name)
+			expression += cType.stringKind
+			if idx != len(components)-1 {
+				expression += ","
+			}
+		}
+		expression += ")"
+		typ.Kind = reflect.Struct
+		typ.Type = reflect.StructOf(fields)
+		typ.TupleElems = elems
+		typ.TupleRawNames = names
+		typ.T = TupleTy
+		typ.stringKind = expression
 	case "function":
 		typ.Kind = reflect.Array
 		typ.T = FunctionTy
@@ -173,28 +223,82 @@ func (t Type) String() (out string) {
 func (t Type) pack(v reflect.Value) ([]byte, error) {
 	// dereference pointer first if it's a pointer
 	v = indirect(v)
-
 	if err := typeCheck(t, v); err != nil {
 		return nil, err
 	}
 
-	if t.T == SliceTy || t.T == ArrayTy {
-		var packed []byte
+	switch t.T {
+	case SliceTy, ArrayTy:
+		var ret []byte
 
+		if t.requiresLengthPrefix() {
+			// append length
+			ret = append(ret, packNum(reflect.ValueOf(v.Len()))...)
+		}
+
+		// calculate offset if any
+		offset := 0
+		offsetReq := isDynamicType(*t.Elem)
+		if offsetReq {
+			offset = getTypeSize(*t.Elem) * v.Len()
+		}
+		var tail []byte
 		for i := 0; i < v.Len(); i++ {
 			val, err := t.Elem.pack(v.Index(i))
 			if err != nil {
 				return nil, err
 			}
-			packed = append(packed, val...)
+			if !offsetReq {
+				ret = append(ret, val...)
+				continue
+			}
+			ret = append(ret, packNum(reflect.ValueOf(offset))...)
+			offset += len(val)
+			tail = append(tail, val...)
 		}
-		if t.T == SliceTy {
-			return packBytesSlice(packed, v.Len()), nil
-		} else if t.T == ArrayTy {
-			return packed, nil
+		return append(ret, tail...), nil
+	case TupleTy:
+		// (T1,...,Tk) for k >= 0 and any types T1, …, Tk
+		// enc(X) = head(X(1)) ... head(X(k)) tail(X(1)) ... tail(X(k))
+		// where X = (X(1), ..., X(k)) and head and tail are defined for Ti being a static
+		// type as
+		//     head(X(i)) = enc(X(i)) and tail(X(i)) = "" (the empty string)
+		// and as
+		//     head(X(i)) = enc(len(head(X(1)) ... head(X(k)) tail(X(1)) ... tail(X(i-1))))
+		//     tail(X(i)) = enc(X(i))
+		// otherwise, i.e. if Ti is a dynamic type.
+		fieldmap, err := mapArgNamesToStructFields(t.TupleRawNames, v)
+		if err != nil {
+			return nil, err
 		}
+		// Calculate prefix occupied size.
+		offset := 0
+		for _, elem := range t.TupleElems {
+			offset += getTypeSize(*elem)
+		}
+		var ret, tail []byte
+		for i, elem := range t.TupleElems {
+			field := v.FieldByName(fieldmap[t.TupleRawNames[i]])
+			if !field.IsValid() {
+				return nil, fmt.Errorf("field %s for tuple not found in the given struct", t.TupleRawNames[i])
+			}
+			val, err := elem.pack(field)
+			if err != nil {
+				return nil, err
+			}
+			if isDynamicType(*elem) {
+				ret = append(ret, packNum(reflect.ValueOf(offset))...)
+				tail = append(tail, val...)
+				offset += len(val)
+			} else {
+				ret = append(ret, val...)
+			}
+		}
+		return append(ret, tail...), nil
+
+	default:
+		return packElement(t, v), nil
 	}
-	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
@@ -202,3 +306,47 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 func (t Type) requiresLengthPrefix() bool {
 	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }
+
+// isDynamicType returns true if the type is dynamic.
+// The following types are called “dynamic”:
+// * bytes
+// * string
+// * T[] for any T
+// * T[k] for any dynamic T and any k >= 0
+// * (T1,...,Tk) if Ti is dynamic for some 1 <= i <= k
+func isDynamicType(t Type) bool {
+	if t.T == TupleTy {
+		for _, elem := range t.TupleElems {
+			if isDynamicType(*elem) {
+				return true
+			}
+		}
+		return false
+	}
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem))
+}
+
+// getTypeSize returns the size that this type needs to occupy.
+// We distinguish static and dynamic types. Static types are encoded in-place
+// and dynamic types are encoded at a separately allocated location after the
+// current block.
+// So for a static variable, the size returned represents the size that the
+// variable actually occupies.
+// For a dynamic variable, the returned size is fixed 32 bytes, which is used
+// to store the location reference for actual value storage.
+func getTypeSize(t Type) int {
+	if t.T == ArrayTy && !isDynamicType(*t.Elem) {
+		// Recursively calculate type size if it is a nested array
+		if t.Elem.T == ArrayTy {
+			return t.Size * getTypeSize(*t.Elem)
+		}
+		return t.Size * 32
+	} else if t.T == TupleTy && !isDynamicType(t) {
+		total := 0
+		for _, elem := range t.TupleElems {
+			total += getTypeSize(*elem)
+		}
+		return total
+	}
+	return 32
+}

accounts/abi/type_test.go

@@ -32,72 +32,75 @@ type typeWithoutStringer Type
 // Tests that all allowed types get recognized by the type parser.
 func TestTypeRegexp(t *testing.T) {
 	tests := []struct {
-		blob string
-		kind Type
+		blob       string
+		components []ArgumentMarshaling
+		kind       Type
 	}{
-		{"bool", Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}},
-		{"bool[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}},
-		{"bool[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}},
-		{"bool[2][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}},
-		{"bool[][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}},
-		{"bool[][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}},
-		{"bool[2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}},
-		{"bool[2][][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}},
-		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
-		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
-		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
-		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
-		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
-		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
-		{"bytes32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
-		{"bytes32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
-		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
-		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
-		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"bool", nil, Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}},
+		{"bool[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}},
+		{"bool[2]", nil, Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool[2][]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}},
+		{"bool[][]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}},
+		{"bool[][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}},
+		{"bool[2][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}},
+		{"bool[2][][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}},
+		{"bool[2][2][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
+		{"bool[][][]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
+		{"bool[][2][]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
+		{"int8", nil, Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int16", nil, Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int32", nil, Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int64", nil, Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int256", nil, Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", nil, Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"uint8", nil, Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint16", nil, Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint32", nil, Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint64", nil, Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint256", nil, Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"bytes32", nil, Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
+		{"bytes[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
+		{"bytes[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
+		{"bytes32[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
+		{"bytes32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
+		{"string", nil, Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
+		{"string[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
+		{"string[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
+		{"address", nil, Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
-		// {"fixed", Type{}},
-		// {"fixed128x128", Type{}},
-		// {"fixed[]", Type{}},
-		// {"fixed[2]", Type{}},
-		// {"fixed128x128[]", Type{}},
-		// {"fixed128x128[2]", Type{}},
+		// {"fixed", nil, Type{}},
+		// {"fixed128x128", nil, Type{}},
+		// {"fixed[]", nil, Type{}},
+		// {"fixed[2]", nil, Type{}},
+		// {"fixed128x128[]", nil, Type{}},
+		// {"fixed128x128[2]", nil, Type{}},
+		{"tuple", []ArgumentMarshaling{{Name: "a", Type: "int64"}}, Type{Kind: reflect.Struct, T: TupleTy, Type: reflect.TypeOf(struct{ A int64 }{}), stringKind: "(int64)",
+			TupleElems: []*Type{{Kind: reflect.Int64, T: IntTy, Type: reflect.TypeOf(int64(0)), Size: 64, stringKind: "int64"}}, TupleRawNames: []string{"a"}}},
 	}
 
 	for _, tt := range tests {
-		typ, err := NewType(tt.blob)
+		typ, err := NewType(tt.blob, tt.components)
 		if err != nil {
 			t.Errorf("type %q: failed to parse type string: %v", tt.blob, err)
 		}
@@ -109,154 +112,170 @@ func TestTypeRegexp(t *testing.T) {
 
 func TestTypeCheck(t *testing.T) {
 	for i, test := range []struct {
-		typ   string
-		input interface{}
-		err   string
+		typ        string
+		components []ArgumentMarshaling
+		input      interface{}
+		err        string
 	}{
-		{"uint", big.NewInt(1), "unsupported arg type: uint"},
-		{"int", big.NewInt(1), "unsupported arg type: int"},
-		{"uint256", big.NewInt(1), ""},
-		{"uint256[][3][]", [][3][]*big.Int{{{}}}, ""},
-		{"uint256[][][3]", [3][][]*big.Int{{{}}}, ""},
-		{"uint256[3][][]", [][][3]*big.Int{{{}}}, ""},
-		{"uint256[3][3][3]", [3][3][3]*big.Int{{{}}}, ""},
-		{"uint8[][]", [][]uint8{}, ""},
-		{"int256", big.NewInt(1), ""},
-		{"uint8", uint8(1), ""},
-		{"uint16", uint16(1), ""},
-		{"uint32", uint32(1), ""},
-		{"uint64", uint64(1), ""},
-		{"int8", int8(1), ""},
-		{"int16", int16(1), ""},
-		{"int32", int32(1), ""},
-		{"int64", int64(1), ""},
-		{"uint24", big.NewInt(1), ""},
-		{"uint40", big.NewInt(1), ""},
-		{"uint48", big.NewInt(1), ""},
-		{"uint56", big.NewInt(1), ""},
-		{"uint72", big.NewInt(1), ""},
-		{"uint80", big.NewInt(1), ""},
-		{"uint88", big.NewInt(1), ""},
-		{"uint96", big.NewInt(1), ""},
-		{"uint104", big.NewInt(1), ""},
-		{"uint112", big.NewInt(1), ""},
-		{"uint120", big.NewInt(1), ""},
-		{"uint128", big.NewInt(1), ""},
-		{"uint136", big.NewInt(1), ""},
-		{"uint144", big.NewInt(1), ""},
-		{"uint152", big.NewInt(1), ""},
-		{"uint160", big.NewInt(1), ""},
-		{"uint168", big.NewInt(1), ""},
-		{"uint176", big.NewInt(1), ""},
-		{"uint184", big.NewInt(1), ""},
-		{"uint192", big.NewInt(1), ""},
-		{"uint200", big.NewInt(1), ""},
-		{"uint208", big.NewInt(1), ""},
-		{"uint216", big.NewInt(1), ""},
-		{"uint224", big.NewInt(1), ""},
-		{"uint232", big.NewInt(1), ""},
-		{"uint240", big.NewInt(1), ""},
-		{"uint248", big.NewInt(1), ""},
-		{"int24", big.NewInt(1), ""},
-		{"int40", big.NewInt(1), ""},
-		{"int48", big.NewInt(1), ""},
-		{"int56", big.NewInt(1), ""},
-		{"int72", big.NewInt(1), ""},
-		{"int80", big.NewInt(1), ""},
-		{"int88", big.NewInt(1), ""},
-		{"int96", big.NewInt(1), ""},
-		{"int104", big.NewInt(1), ""},
-		{"int112", big.NewInt(1), ""},
-		{"int120", big.NewInt(1), ""},
-		{"int128", big.NewInt(1), ""},
-		{"int136", big.NewInt(1), ""},
-		{"int144", big.NewInt(1), ""},
-		{"int152", big.NewInt(1), ""},
-		{"int160", big.NewInt(1), ""},
-		{"int168", big.NewInt(1), ""},
-		{"int176", big.NewInt(1), ""},
-		{"int184", big.NewInt(1), ""},
-		{"int192", big.NewInt(1), ""},
-		{"int200", big.NewInt(1), ""},
-		{"int208", big.NewInt(1), ""},
-		{"int216", big.NewInt(1), ""},
-		{"int224", big.NewInt(1), ""},
-		{"int232", big.NewInt(1), ""},
-		{"int240", big.NewInt(1), ""},
-		{"int248", big.NewInt(1), ""},
-		{"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"},
-		{"uint8", uint16(1), "abi: cannot use uint16 as type uint8 as argument"},
-		{"uint8", uint32(1), "abi: cannot use uint32 as type uint8 as argument"},
-		{"uint8", uint64(1), "abi: cannot use uint64 as type uint8 as argument"},
-		{"uint8", int8(1), "abi: cannot use int8 as type uint8 as argument"},
-		{"uint8", int16(1), "abi: cannot use int16 as type uint8 as argument"},
-		{"uint8", int32(1), "abi: cannot use int32 as type uint8 as argument"},
-		{"uint8", int64(1), "abi: cannot use int64 as type uint8 as argument"},
-		{"uint16", uint16(1), ""},
-		{"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"},
-		{"uint16[]", []uint16{1, 2, 3}, ""},
-		{"uint16[]", [3]uint16{1, 2, 3}, ""},
-		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"},
-		{"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"},
-		{"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
-		{"uint16[3]", []uint16{1, 2, 3}, ""},
-		{"uint16[3]", []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
-		{"address[]", []common.Address{{1}}, ""},
-		{"address[1]", []common.Address{{1}}, ""},
-		{"address[1]", [1]common.Address{{1}}, ""},
-		{"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
-		{"bytes32", [32]byte{}, ""},
-		{"bytes31", [31]byte{}, ""},
-		{"bytes30", [30]byte{}, ""},
-		{"bytes29", [29]byte{}, ""},
-		{"bytes28", [28]byte{}, ""},
-		{"bytes27", [27]byte{}, ""},
-		{"bytes26", [26]byte{}, ""},
-		{"bytes25", [25]byte{}, ""},
-		{"bytes24", [24]byte{}, ""},
-		{"bytes23", [23]byte{}, ""},
-		{"bytes22", [22]byte{}, ""},
-		{"bytes21", [21]byte{}, ""},
-		{"bytes20", [20]byte{}, ""},
-		{"bytes19", [19]byte{}, ""},
-		{"bytes18", [18]byte{}, ""},
-		{"bytes17", [17]byte{}, ""},
-		{"bytes16", [16]byte{}, ""},
-		{"bytes15", [15]byte{}, ""},
-		{"bytes14", [14]byte{}, ""},
-		{"bytes13", [13]byte{}, ""},
-		{"bytes12", [12]byte{}, ""},
-		{"bytes11", [11]byte{}, ""},
-		{"bytes10", [10]byte{}, ""},
-		{"bytes9", [9]byte{}, ""},
-		{"bytes8", [8]byte{}, ""},
-		{"bytes7", [7]byte{}, ""},
-		{"bytes6", [6]byte{}, ""},
-		{"bytes5", [5]byte{}, ""},
-		{"bytes4", [4]byte{}, ""},
-		{"bytes3", [3]byte{}, ""},
-		{"bytes2", [2]byte{}, ""},
-		{"bytes1", [1]byte{}, ""},
-		{"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
-		{"bytes32", common.Hash{1}, ""},
-		{"bytes31", common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"},
-		{"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"},
-		{"bytes", []byte{0, 1}, ""},
-		{"bytes", [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
-		{"bytes", common.Hash{1}, "abi: cannot use array as type slice as argument"},
-		{"string", "hello world", ""},
-		{"string", string(""), ""},
-		{"string", []byte{}, "abi: cannot use slice as type string as argument"},
-		{"bytes32[]", [][32]byte{{}}, ""},
-		{"function", [24]byte{}, ""},
-		{"bytes20", common.Address{}, ""},
-		{"address", [20]byte{}, ""},
-		{"address", common.Address{}, ""},
-		{"bytes32[]]", "", "invalid arg type in abi"},
-		{"invalidType", "", "unsupported arg type: invalidType"},
-		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
+		{"uint", nil, big.NewInt(1), "unsupported arg type: uint"},
+		{"int", nil, big.NewInt(1), "unsupported arg type: int"},
+		{"uint256", nil, big.NewInt(1), ""},
+		{"uint256[][3][]", nil, [][3][]*big.Int{{{}}}, ""},
+		{"uint256[][][3]", nil, [3][][]*big.Int{{{}}}, ""},
+		{"uint256[3][][]", nil, [][][3]*big.Int{{{}}}, ""},
+		{"uint256[3][3][3]", nil, [3][3][3]*big.Int{{{}}}, ""},
+		{"uint8[][]", nil, [][]uint8{}, ""},
+		{"int256", nil, big.NewInt(1), ""},
+		{"uint8", nil, uint8(1), ""},
+		{"uint16", nil, uint16(1), ""},
+		{"uint32", nil, uint32(1), ""},
+		{"uint64", nil, uint64(1), ""},
+		{"int8", nil, int8(1), ""},
+		{"int16", nil, int16(1), ""},
+		{"int32", nil, int32(1), ""},
+		{"int64", nil, int64(1), ""},
+		{"uint24", nil, big.NewInt(1), ""},
+		{"uint40", nil, big.NewInt(1), ""},
+		{"uint48", nil, big.NewInt(1), ""},
+		{"uint56", nil, big.NewInt(1), ""},
+		{"uint72", nil, big.NewInt(1), ""},
+		{"uint80", nil, big.NewInt(1), ""},
+		{"uint88", nil, big.NewInt(1), ""},
+		{"uint96", nil, big.NewInt(1), ""},
+		{"uint104", nil, big.NewInt(1), ""},
+		{"uint112", nil, big.NewInt(1), ""},
+		{"uint120", nil, big.NewInt(1), ""},
+		{"uint128", nil, big.NewInt(1), ""},
+		{"uint136", nil, big.NewInt(1), ""},
+		{"uint144", nil, big.NewInt(1), ""},
+		{"uint152", nil, big.NewInt(1), ""},
+		{"uint160", nil, big.NewInt(1), ""},
+		{"uint168", nil, big.NewInt(1), ""},
+		{"uint176", nil, big.NewInt(1), ""},
+		{"uint184", nil, big.NewInt(1), ""},
+		{"uint192", nil, big.NewInt(1), ""},
+		{"uint200", nil, big.NewInt(1), ""},
+		{"uint208", nil, big.NewInt(1), ""},
+		{"uint216", nil, big.NewInt(1), ""},
+		{"uint224", nil, big.NewInt(1), ""},
+		{"uint232", nil, big.NewInt(1), ""},
+		{"uint240", nil, big.NewInt(1), ""},
+		{"uint248", nil, big.NewInt(1), ""},
+		{"int24", nil, big.NewInt(1), ""},
+		{"int40", nil, big.NewInt(1), ""},
+		{"int48", nil, big.NewInt(1), ""},
+		{"int56", nil, big.NewInt(1), ""},
+		{"int72", nil, big.NewInt(1), ""},
+		{"int80", nil, big.NewInt(1), ""},
+		{"int88", nil, big.NewInt(1), ""},
+		{"int96", nil, big.NewInt(1), ""},
+		{"int104", nil, big.NewInt(1), ""},
+		{"int112", nil, big.NewInt(1), ""},
+		{"int120", nil, big.NewInt(1), ""},
+		{"int128", nil, big.NewInt(1), ""},
+		{"int136", nil, big.NewInt(1), ""},
+		{"int144", nil, big.NewInt(1), ""},
+		{"int152", nil, big.NewInt(1), ""},
+		{"int160", nil, big.NewInt(1), ""},
+		{"int168", nil, big.NewInt(1), ""},
+		{"int176", nil, big.NewInt(1), ""},
+		{"int184", nil, big.NewInt(1), ""},
+		{"int192", nil, big.NewInt(1), ""},
+		{"int200", nil, big.NewInt(1), ""},
+		{"int208", nil, big.NewInt(1), ""},
+		{"int216", nil, big.NewInt(1), ""},
+		{"int224", nil, big.NewInt(1), ""},
+		{"int232", nil, big.NewInt(1), ""},
+		{"int240", nil, big.NewInt(1), ""},
+		{"int248", nil, big.NewInt(1), ""},
+		{"uint30", nil, uint8(1), "abi: cannot use uint8 as type ptr as argument"},
+		{"uint8", nil, uint16(1), "abi: cannot use uint16 as type uint8 as argument"},
+		{"uint8", nil, uint32(1), "abi: cannot use uint32 as type uint8 as argument"},
+		{"uint8", nil, uint64(1), "abi: cannot use uint64 as type uint8 as argument"},
+		{"uint8", nil, int8(1), "abi: cannot use int8 as type uint8 as argument"},
+		{"uint8", nil, int16(1), "abi: cannot use int16 as type uint8 as argument"},
+		{"uint8", nil, int32(1), "abi: cannot use int32 as type uint8 as argument"},
+		{"uint8", nil, int64(1), "abi: cannot use int64 as type uint8 as argument"},
+		{"uint16", nil, uint16(1), ""},
+		{"uint16", nil, uint8(1), "abi: cannot use uint8 as type uint16 as argument"},
+		{"uint16[]", nil, []uint16{1, 2, 3}, ""},
+		{"uint16[]", nil, [3]uint16{1, 2, 3}, ""},
+		{"uint16[]", nil, []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"},
+		{"uint16[3]", nil, [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"},
+		{"uint16[3]", nil, [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
+		{"uint16[3]", nil, []uint16{1, 2, 3}, ""},
+		{"uint16[3]", nil, []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
+		{"address[]", nil, []common.Address{{1}}, ""},
+		{"address[1]", nil, []common.Address{{1}}, ""},
+		{"address[1]", nil, [1]common.Address{{1}}, ""},
+		{"address[2]", nil, [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
+		{"bytes32", nil, [32]byte{}, ""},
+		{"bytes31", nil, [31]byte{}, ""},
+		{"bytes30", nil, [30]byte{}, ""},
+		{"bytes29", nil, [29]byte{}, ""},
+		{"bytes28", nil, [28]byte{}, ""},
+		{"bytes27", nil, [27]byte{}, ""},
+		{"bytes26", nil, [26]byte{}, ""},
+		{"bytes25", nil, [25]byte{}, ""},
+		{"bytes24", nil, [24]byte{}, ""},
+		{"bytes23", nil, [23]byte{}, ""},
+		{"bytes22", nil, [22]byte{}, ""},
+		{"bytes21", nil, [21]byte{}, ""},
+		{"bytes20", nil, [20]byte{}, ""},
+		{"bytes19", nil, [19]byte{}, ""},
+		{"bytes18", nil, [18]byte{}, ""},
+		{"bytes17", nil, [17]byte{}, ""},
+		{"bytes16", nil, [16]byte{}, ""},
+		{"bytes15", nil, [15]byte{}, ""},
+		{"bytes14", nil, [14]byte{}, ""},
+		{"bytes13", nil, [13]byte{}, ""},
+		{"bytes12", nil, [12]byte{}, ""},
+		{"bytes11", nil, [11]byte{}, ""},
+		{"bytes10", nil, [10]byte{}, ""},
+		{"bytes9", nil, [9]byte{}, ""},
+		{"bytes8", nil, [8]byte{}, ""},
+		{"bytes7", nil, [7]byte{}, ""},
+		{"bytes6", nil, [6]byte{}, ""},
+		{"bytes5", nil, [5]byte{}, ""},
+		{"bytes4", nil, [4]byte{}, ""},
+		{"bytes3", nil, [3]byte{}, ""},
+		{"bytes2", nil, [2]byte{}, ""},
+		{"bytes1", nil, [1]byte{}, ""},
+		{"bytes32", nil, [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
+		{"bytes32", nil, common.Hash{1}, ""},
+		{"bytes31", nil, common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"},
+		{"bytes31", nil, [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"},
+		{"bytes", nil, []byte{0, 1}, ""},
+		{"bytes", nil, [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
+		{"bytes", nil, common.Hash{1}, "abi: cannot use array as type slice as argument"},
+		{"string", nil, "hello world", ""},
+		{"string", nil, string(""), ""},
+		{"string", nil, []byte{}, "abi: cannot use slice as type string as argument"},
+		{"bytes32[]", nil, [][32]byte{{}}, ""},
+		{"function", nil, [24]byte{}, ""},
+		{"bytes20", nil, common.Address{}, ""},
+		{"address", nil, [20]byte{}, ""},
+		{"address", nil, common.Address{}, ""},
+		{"bytes32[]]", nil, "", "invalid arg type in abi"},
+		{"invalidType", nil, "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", nil, "", "unsupported arg type: invalidSlice"},
+		// simple tuple
+		{"tuple", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, struct {
+			A *big.Int
+			B *big.Int
+		}{}, ""},
+		// tuple slice
+		{"tuple[]", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, []struct {
+			A *big.Int
+			B *big.Int
+		}{}, ""},
+		// tuple array
+		{"tuple[2]", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, []struct {
+			A *big.Int
+			B *big.Int
+		}{{big.NewInt(0), big.NewInt(0)}, {big.NewInt(0), big.NewInt(0)}}, ""},
 	} {
-		typ, err := NewType(test.typ)
+		typ, err := NewType(test.typ, test.components)
 		if err != nil && len(test.err) == 0 {
 			t.Fatal("unexpected parse error:", err)
 		} else if err != nil && len(test.err) != 0 {

accounts/abi/unpack.go

@@ -25,8 +25,17 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+var (
+	maxUint256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
+		big.NewInt(-1))
+	maxInt256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
+		big.NewInt(-1))
+)
+
 // reads the integer based on its kind
-func readInteger(kind reflect.Kind, b []byte) interface{} {
+func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
 		return b[len(b)-1]
@@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	case reflect.Int64:
 		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
 	default:
-		return new(big.Int).SetBytes(b)
+		// the only case lefts for integer is int256/uint256.
+		// big.SetBytes can't tell if a number is negative, positive on itself.
+		// On EVM, if the returned number > max int256, it is negative.
+		ret := new(big.Int).SetBytes(b)
+		if typ == UintTy {
+			return ret
+		}
+
+		if ret.Cmp(maxInt256) > 0 {
+			ret.Add(maxUint256, big.NewInt(0).Neg(ret))
+			ret.Add(ret, big.NewInt(1))
+			ret.Neg(ret)
+		}
+		return ret
 	}
 }
 
@@ -93,17 +115,6 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) {
 
 }
 
-func getFullElemSize(elem *Type) int {
-	//all other should be counted as 32 (slices have pointers to respective elements)
-	size := 32
-	//arrays wrap it, each element being the same size
-	for elem.T == ArrayTy {
-		size *= elem.Size
-		elem = elem.Elem
-	}
-	return size
-}
-
 // iteratively unpack elements
 func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
 	if size < 0 {
@@ -128,13 +139,9 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 
 	// Arrays have packed elements, resulting in longer unpack steps.
 	// Slices have just 32 bytes per element (pointing to the contents).
-	elemSize := 32
-	if t.T == ArrayTy {
-		elemSize = getFullElemSize(t.Elem)
-	}
+	elemSize := getTypeSize(*t.Elem)
 
 	for i, j := start, 0; j < size; i, j = i+elemSize, j+1 {
-
 		inter, err := toGoType(i, *t.Elem, output)
 		if err != nil {
 			return nil, err
@@ -148,6 +155,36 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 	return refSlice.Interface(), nil
 }
 
+func forTupleUnpack(t Type, output []byte) (interface{}, error) {
+	retval := reflect.New(t.Type).Elem()
+	virtualArgs := 0
+	for index, elem := range t.TupleElems {
+		marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output)
+		if elem.T == ArrayTy && !isDynamicType(*elem) {
+			// If we have a static array, like [3]uint256, these are coded as
+			// just like uint256,uint256,uint256.
+			// This means that we need to add two 'virtual' arguments when
+			// we count the index from now on.
+			//
+			// Array values nested multiple levels deep are also encoded inline:
+			// [2][3]uint256: uint256,uint256,uint256,uint256,uint256,uint256
+			//
+			// Calculate the full array size to get the correct offset for the next argument.
+			// Decrement it by 1, as the normal index increment is still applied.
+			virtualArgs += getTypeSize(*elem)/32 - 1
+		} else if elem.T == TupleTy && !isDynamicType(*elem) {
+			// If we have a static tuple, like (uint256, bool, uint256), these are
+			// coded as just like uint256,bool,uint256
+			virtualArgs += getTypeSize(*elem)/32 - 1
+		}
+		if err != nil {
+			return nil, err
+		}
+		retval.Field(index).Set(reflect.ValueOf(marshalledValue))
+	}
+	return retval.Interface(), nil
+}
+
 // toGoType parses the output bytes and recursively assigns the value of these bytes
 // into a go type with accordance with the ABI spec.
 func toGoType(index int, t Type, output []byte) (interface{}, error) {
@@ -156,14 +193,14 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	}
 
 	var (
-		returnOutput []byte
-		begin, end   int
-		err          error
+		returnOutput  []byte
+		begin, length int
+		err           error
 	)
 
 	// if we require a length prefix, find the beginning word and size returned.
 	if t.requiresLengthPrefix() {
-		begin, end, err = lengthPrefixPointsTo(index, output)
+		begin, length, err = lengthPrefixPointsTo(index, output)
 		if err != nil {
 			return nil, err
 		}
@@ -172,14 +209,28 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	}
 
 	switch t.T {
+	case TupleTy:
+		if isDynamicType(t) {
+			begin, err := tuplePointsTo(index, output)
+			if err != nil {
+				return nil, err
+			}
+			return forTupleUnpack(t, output[begin:])
+		} else {
+			return forTupleUnpack(t, output[index:])
+		}
 	case SliceTy:
-		return forEachUnpack(t, output, begin, end)
+		return forEachUnpack(t, output[begin:], 0, length)
 	case ArrayTy:
-		return forEachUnpack(t, output, index, t.Size)
+		if isDynamicType(*t.Elem) {
+			offset := int64(binary.BigEndian.Uint64(returnOutput[len(returnOutput)-8:]))
+			return forEachUnpack(t, output[offset:], 0, t.Size)
+		}
+		return forEachUnpack(t, output[index:], 0, t.Size)
 	case StringTy: // variable arrays are written at the end of the return bytes
-		return string(output[begin : begin+end]), nil
+		return string(output[begin : begin+length]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Kind, returnOutput), nil
+		return readInteger(t.T, t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:
@@ -187,7 +238,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	case HashTy:
 		return common.BytesToHash(returnOutput), nil
 	case BytesTy:
-		return output[begin : begin+end], nil
+		return output[begin : begin+length], nil
 	case FixedBytesTy:
 		return readFixedBytes(t, returnOutput)
 	case FunctionTy:
@@ -228,3 +279,17 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err
 	length = int(lengthBig.Uint64())
 	return
 }
+
+// tuplePointsTo resolves the location reference for dynamic tuple.
+func tuplePointsTo(index int, output []byte) (start int, err error) {
+	offset := big.NewInt(0).SetBytes(output[index : index+32])
+	outputLen := big.NewInt(int64(len(output)))
+
+	if offset.Cmp(big.NewInt(int64(len(output)))) > 0 {
+		return 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", offset, outputLen)
+	}
+	if offset.BitLen() > 63 {
+		return 0, fmt.Errorf("abi offset larger than int64: %v", offset)
+	}
+	return int(offset.Uint64()), nil
+}

accounts/abi/unpack_test.go

@@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: big.NewInt(1),
 	},
+	{
+		def:  `[{"type": "int256"}]`,
+		enc:  "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		want: big.NewInt(-1),
+	},
 	{
 		def:  `[{"type": "address"}]`,
 		enc:  "0000000000000000000000000100000000000000000000000000000000000000",
@@ -168,9 +173,14 @@ var unpackTests = []unpackTest{
 	// multi dimensional, if these pass, all types that don't require length prefix should pass
 	{
 		def:  `[{"type": "uint8[][]"}]`,
-		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000E0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
 		want: [][]uint8{{1, 2}, {1, 2}},
 	},
+	{
+		def:  `[{"type": "uint8[][]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		want: [][]uint8{{1, 2}, {1, 2, 3}},
+	},
 	{
 		def:  `[{"type": "uint8[2][2]"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
@@ -178,7 +188,7 @@ var unpackTests = []unpackTest{
 	},
 	{
 		def:  `[{"type": "uint8[][2]"}]`,
-		enc:  "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001",
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001",
 		want: [2][]uint8{{1}, {1}},
 	},
 	{
@@ -186,6 +196,11 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
 		want: [][2]uint8{{1, 2}},
 	},
+	{
+		def:  `[{"type": "uint8[2][]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [][2]uint8{{1, 2}, {1, 2}},
+	},
 	{
 		def:  `[{"type": "uint16[]"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
@@ -231,6 +246,26 @@ var unpackTests = []unpackTest{
 		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
 		want: [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
 	},
+	{
+		def:  `[{"type": "string[4]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000000548656c6c6f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005576f726c64000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b476f2d657468657265756d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000",
+		want: [4]string{"Hello", "World", "Go-ethereum", "Ethereum"},
+	},
+	{
+		def:  `[{"type": "string[]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b676f2d657468657265756d000000000000000000000000000000000000000000",
+		want: []string{"Ethereum", "go-ethereum"},
+	},
+	{
+		def:  `[{"type": "bytes[]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000003f0f0f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003f0f0f00000000000000000000000000000000000000000000000000000000000",
+		want: [][]byte{{0xf0, 0xf0, 0xf0}, {0xf0, 0xf0, 0xf0}},
+	},
+	{
+		def:  `[{"type": "uint256[2][][]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000c8000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000003e80000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000c8000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000003e8",
+		want: [][][2]*big.Int{{{big.NewInt(1), big.NewInt(200)}, {big.NewInt(1), big.NewInt(1000)}}, {{big.NewInt(1), big.NewInt(200)}, {big.NewInt(1), big.NewInt(1000)}}},
+	},
 	{
 		def:  `[{"type": "int8[]"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
@@ -290,6 +325,53 @@ var unpackTests = []unpackTest{
 			Int2 *big.Int
 		}{big.NewInt(1), big.NewInt(2)},
 	},
+	{
+		def: `[{"name":"int_one","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			IntOne *big.Int
+		}{big.NewInt(1)},
+	},
+	{
+		def: `[{"name":"int__one","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			IntOne *big.Int
+		}{big.NewInt(1)},
+	},
+	{
+		def: `[{"name":"int_one_","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			IntOne *big.Int
+		}{big.NewInt(1)},
+	},
+	{
+		def: `[{"name":"int_one","type":"int256"}, {"name":"intone","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			IntOne *big.Int
+			Intone *big.Int
+		}{big.NewInt(1), big.NewInt(2)},
+	},
+	{
+		def: `[{"name":"___","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			IntOne *big.Int
+			Intone *big.Int
+		}{},
+		err: "abi: purely underscored output cannot unpack to struct",
+	},
+	{
+		def: `[{"name":"int_one","type":"int256"},{"name":"IntOne","type":"int256"}]`,
+		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: struct {
+			Int1 *big.Int
+			Int2 *big.Int
+		}{},
+		err: "abi: multiple outputs mapping to the same struct field 'IntOne'",
+	},
 	{
 		def: `[{"name":"int","type":"int256"},{"name":"Int","type":"int256"}]`,
 		enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
@@ -354,6 +436,55 @@ func TestUnpack(t *testing.T) {
 	}
 }
 
+func TestUnpackSetDynamicArrayOutput(t *testing.T) {
+	abi, err := JSON(strings.NewReader(`[{"constant":true,"inputs":[],"name":"testDynamicFixedBytes15","outputs":[{"name":"","type":"bytes15[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"testDynamicFixedBytes32","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"}]`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var (
+		marshalledReturn32 = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000230783132333435363738393000000000000000000000000000000000000000003078303938373635343332310000000000000000000000000000000000000000")
+		marshalledReturn15 = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000230783031323334350000000000000000000000000000000000000000000000003078393837363534000000000000000000000000000000000000000000000000")
+
+		out32 [][32]byte
+		out15 [][15]byte
+	)
+
+	// test 32
+	err = abi.Unpack(&out32, "testDynamicFixedBytes32", marshalledReturn32)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out32) != 2 {
+		t.Fatalf("expected array with 2 values, got %d", len(out32))
+	}
+	expected := common.Hex2Bytes("3078313233343536373839300000000000000000000000000000000000000000")
+	if !bytes.Equal(out32[0][:], expected) {
+		t.Errorf("expected %x, got %x\n", expected, out32[0])
+	}
+	expected = common.Hex2Bytes("3078303938373635343332310000000000000000000000000000000000000000")
+	if !bytes.Equal(out32[1][:], expected) {
+		t.Errorf("expected %x, got %x\n", expected, out32[1])
+	}
+
+	// test 15
+	err = abi.Unpack(&out15, "testDynamicFixedBytes32", marshalledReturn15)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out15) != 2 {
+		t.Fatalf("expected array with 2 values, got %d", len(out15))
+	}
+	expected = common.Hex2Bytes("307830313233343500000000000000")
+	if !bytes.Equal(out15[0][:], expected) {
+		t.Errorf("expected %x, got %x\n", expected, out15[0])
+	}
+	expected = common.Hex2Bytes("307839383736353400000000000000")
+	if !bytes.Equal(out15[1][:], expected) {
+		t.Errorf("expected %x, got %x\n", expected, out15[1])
+	}
+}
+
 type methodMultiOutput struct {
 	Int    *big.Int
 	String string
@@ -381,6 +512,11 @@ func TestMethodMultiReturn(t *testing.T) {
 		Int    *big.Int
 	}
 
+	newInterfaceSlice := func(len int) interface{} {
+		slice := make([]interface{}, len)
+		return &slice
+	}
+
 	abi, data, expected := methodMultiReturn(require.New(t))
 	bigint := new(big.Int)
 	var testCases = []struct {
@@ -408,6 +544,16 @@ func TestMethodMultiReturn(t *testing.T) {
 		&[2]interface{}{&expected.Int, &expected.String},
 		"",
 		"Can unpack into an array",
+	}, {
+		&[2]interface{}{},
+		&[2]interface{}{expected.Int, expected.String},
+		"",
+		"Can unpack into interface array",
+	}, {
+		newInterfaceSlice(2),
+		&[]interface{}{expected.Int, expected.String},
+		"",
+		"Can unpack into interface slice",
 	}, {
 		&[]interface{}{new(int), new(int)},
 		&[]interface{}{&expected.Int, &expected.String},
@@ -457,6 +603,68 @@ func TestMultiReturnWithArray(t *testing.T) {
 	}
 }
 
+func TestMultiReturnWithStringArray(t *testing.T) {
+	const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]`
+	abi, err := JSON(strings.NewReader(definition))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff := new(bytes.Buffer)
+	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000005c1b78ea0000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000001a055690d9db80000000000000000000000000000ab1257528b3782fb40d7ed5f72e624b744dffb2f00000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001048656c6c6f2c20457468657265756d2100000000000000000000000000000000"))
+	temp, _ := big.NewInt(0).SetString("30000000000000000000", 10)
+	ret1, ret1Exp := new([3]*big.Int), [3]*big.Int{big.NewInt(1545304298), big.NewInt(6), temp}
+	ret2, ret2Exp := new(common.Address), common.HexToAddress("ab1257528b3782fb40d7ed5f72e624b744dffb2f")
+	ret3, ret3Exp := new([2]string), [2]string{"Ethereum", "Hello, Ethereum!"}
+	ret4, ret4Exp := new(bool), false
+	if err := abi.Unpack(&[]interface{}{ret1, ret2, ret3, ret4}, "multi", buff.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(*ret1, ret1Exp) {
+		t.Error("big.Int array result", *ret1, "!= Expected", ret1Exp)
+	}
+	if !reflect.DeepEqual(*ret2, ret2Exp) {
+		t.Error("address result", *ret2, "!= Expected", ret2Exp)
+	}
+	if !reflect.DeepEqual(*ret3, ret3Exp) {
+		t.Error("string array result", *ret3, "!= Expected", ret3Exp)
+	}
+	if !reflect.DeepEqual(*ret4, ret4Exp) {
+		t.Error("bool result", *ret4, "!= Expected", ret4Exp)
+	}
+}
+
+func TestMultiReturnWithStringSlice(t *testing.T) {
+	const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]`
+	abi, err := JSON(strings.NewReader(definition))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff := new(bytes.Buffer)
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // output[0] offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000120")) // output[1] offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // output[0] length
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // output[0][0] offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // output[0][1] offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000008")) // output[0][0] length
+	buff.Write(common.Hex2Bytes("657468657265756d000000000000000000000000000000000000000000000000")) // output[0][0] value
+	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000b")) // output[0][1] length
+	buff.Write(common.Hex2Bytes("676f2d657468657265756d000000000000000000000000000000000000000000")) // output[0][1] value
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // output[1] length
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000064")) // output[1][0] value
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000065")) // output[1][1] value
+	ret1, ret1Exp := new([]string), []string{"ethereum", "go-ethereum"}
+	ret2, ret2Exp := new([]*big.Int), []*big.Int{big.NewInt(100), big.NewInt(101)}
+	if err := abi.Unpack(&[]interface{}{ret1, ret2}, "multi", buff.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(*ret1, ret1Exp) {
+		t.Error("string slice result", *ret1, "!= Expected", ret1Exp)
+	}
+	if !reflect.DeepEqual(*ret2, ret2Exp) {
+		t.Error("uint256 slice result", *ret2, "!= Expected", ret2Exp)
+	}
+}
+
 func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
 	// Similar to TestMultiReturnWithArray, but with a special case in mind:
 	//  values of nested static arrays count towards the size as well, and any element following
@@ -746,6 +954,108 @@ func TestUnmarshal(t *testing.T) {
 	}
 }
 
+func TestUnpackTuple(t *testing.T) {
+	const simpleTuple = `[{"name":"tuple","constant":false,"outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]`
+	abi, err := JSON(strings.NewReader(simpleTuple))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff := new(bytes.Buffer)
+
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // ret[a] = 1
+	buff.Write(common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) // ret[b] = -1
+
+	v := struct {
+		Ret struct {
+			A *big.Int
+			B *big.Int
+		}
+	}{Ret: struct {
+		A *big.Int
+		B *big.Int
+	}{new(big.Int), new(big.Int)}}
+
+	err = abi.Unpack(&v, "tuple", buff.Bytes())
+	if err != nil {
+		t.Error(err)
+	} else {
+		if v.Ret.A.Cmp(big.NewInt(1)) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.Ret.A)
+		}
+		if v.Ret.B.Cmp(big.NewInt(-1)) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", v.Ret.B, -1)
+		}
+	}
+
+	// Test nested tuple
+	const nestedTuple = `[{"name":"tuple","constant":false,"outputs":[
+		{"type":"tuple","name":"s","components":[{"type":"uint256","name":"a"},{"type":"uint256[]","name":"b"},{"type":"tuple[]","name":"c","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]}]},
+		{"type":"tuple","name":"t","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]},
+		{"type":"uint256","name":"a"}
+	]}]`
+
+	abi, err = JSON(strings.NewReader(nestedTuple))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff.Reset()
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // s offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")) // t.X = 0
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // t.Y = 1
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // a = 1
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.A = 1
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000060")) // s.B offset
+	buff.Write(common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000c0")) // s.C offset
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.B length
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.B[0] = 1
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.B[0] = 2
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C length
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.C[0].X
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C[0].Y
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C[1].X
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.C[1].Y
+
+	type T struct {
+		X *big.Int `abi:"x"`
+		Z *big.Int `abi:"y"` // Test whether the abi tag works.
+	}
+
+	type S struct {
+		A *big.Int
+		B []*big.Int
+		C []T
+	}
+
+	type Ret struct {
+		FieldS S `abi:"s"`
+		FieldT T `abi:"t"`
+		A      *big.Int
+	}
+	var ret Ret
+	var expected = Ret{
+		FieldS: S{
+			A: big.NewInt(1),
+			B: []*big.Int{big.NewInt(1), big.NewInt(2)},
+			C: []T{
+				{big.NewInt(1), big.NewInt(2)},
+				{big.NewInt(2), big.NewInt(1)},
+			},
+		},
+		FieldT: T{
+			big.NewInt(0), big.NewInt(1),
+		},
+		A: big.NewInt(1),
+	}
+
+	err = abi.Unpack(&ret, "tuple", buff.Bytes())
+	if err != nil {
+		t.Error(err)
+	}
+	if reflect.DeepEqual(ret, expected) {
+		t.Error("unexpected unpack value")
+	}
+}
+
 func TestOOMMaliciousInput(t *testing.T) {
 	oomTests := []unpackTest{
 		{

accounts/accounts.go

@@ -18,12 +18,14 @@
 package accounts
 
 import (
+	"fmt"
 	"math/big"
 
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"
+	"golang.org/x/crypto/sha3"
 )
 
 // Account represents an Ethereum account located at a specific location defined
@@ -33,6 +35,13 @@ type Account struct {
 	URL     URL            `json:"url"`     // Optional resource locator within a backend
 }
 
+const (
+	MimetypeTextWithValidator = "text/validator"
+	MimetypeTypedData         = "data/typed"
+	MimetypeClique            = "application/x-clique-header"
+	MimetypeTextPlain         = "text/plain"
+)
+
 // Wallet represents a software or hardware wallet that might contain one or more
 // accounts (derived from the same seed).
 type Wallet interface {
@@ -87,8 +96,26 @@ type Wallet interface {
 	// chain state reader.
 	SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
 
-	// SignHash requests the wallet to sign the given hash.
+	// SignData requests the wallet to sign the hash of the given data
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignDataWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignData(account Account, mimeType string, data []byte) ([]byte, error)
+
+	// SignDataWithPassphrase is identical to SignData, but also takes a password
+	// NOTE: there's an chance that an erroneous call might mistake the two strings, and
+	// supply password in the mimetype field, or vice versa. Thus, an implementation
+	// should never echo the mimetype or return the mimetype in the error-response
+	SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error)
+
+	// SignText requests the wallet to sign the hash of a given piece of data, prefixed
+	// by the Ethereum prefix scheme
 	// It looks up the account specified either solely via its address contained within,
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
@@ -98,7 +125,10 @@ type Wallet interface {
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
-	SignHash(account Account, hash []byte) ([]byte, error)
+	SignText(account Account, text []byte) ([]byte, error)
+
+	// SignTextWithPassphrase is identical to Signtext, but also takes a password
+	SignTextWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
 
 	// SignTx requests the wallet to sign the given transaction.
 	//
@@ -113,18 +143,7 @@ type Wallet interface {
 	// the account in a keystore).
 	SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
 
-	// SignHashWithPassphrase requests the wallet to sign the given hash with the
-	// given passphrase as extra authentication information.
-	//
-	// It looks up the account specified either solely via its address contained within,
-	// or optionally with the aid of any location metadata from the embedded URL field.
-	SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
-
-	// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
-	// given passphrase as extra authentication information.
-	//
-	// It looks up the account specified either solely via its address contained within,
-	// or optionally with the aid of any location metadata from the embedded URL field.
+	// SignTxWithPassphrase is identical to SignTx, but also takes a password
 	SignTxWithPassphrase(account Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
 }
 
@@ -148,6 +167,32 @@ type Backend interface {
 	Subscribe(sink chan<- WalletEvent) event.Subscription
 }
 
+// TextHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calulcated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func TextHash(data []byte) []byte {
+	hash, _ := TextAndHash(data)
+	return hash
+}
+
+// TextAndHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calulcated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func TextAndHash(data []byte) ([]byte, string) {
+	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data))
+	hasher := sha3.NewLegacyKeccak256()
+	hasher.Write([]byte(msg))
+	return hasher.Sum(nil), msg
+}
+
 // WalletEventType represents the different event types that can be fired by
 // the wallet subscription subsystem.
 type WalletEventType int

accounts/accounts_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,14 +14,19 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package whisperv5
+package accounts
 
-type Config struct {
-	MaxMessageSize     uint32  `toml:",omitempty"`
-	MinimumAcceptedPOW float64 `toml:",omitempty"`
-}
+import (
+	"bytes"
+	"testing"
 
-var DefaultConfig = Config{
-	MaxMessageSize:     DefaultMaxMessageSize,
-	MinimumAcceptedPOW: DefaultMinimumPoW,
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+func TestTextHash(t *testing.T) {
+	hash := TextHash([]byte("Hello Joe"))
+	want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b")
+	if !bytes.Equal(hash, want) {
+		t.Fatalf("wrong hash: %x", hash)
+	}
 }

accounts/external/backend.go

@@ -0,0 +1,228 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package external
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+)
+
+type ExternalBackend struct {
+	signers []accounts.Wallet
+}
+
+func (eb *ExternalBackend) Wallets() []accounts.Wallet {
+	return eb.signers
+}
+
+func NewExternalBackend(endpoint string) (*ExternalBackend, error) {
+	signer, err := NewExternalSigner(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &ExternalBackend{
+		signers: []accounts.Wallet{signer},
+	}, nil
+}
+
+func (eb *ExternalBackend) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		<-quit
+		return nil
+	})
+}
+
+// ExternalSigner provides an API to interact with an external signer (clef)
+// It proxies request to the external signer while forwarding relevant
+// request headers
+type ExternalSigner struct {
+	client   *rpc.Client
+	endpoint string
+	status   string
+	cacheMu  sync.RWMutex
+	cache    []accounts.Account
+}
+
+func NewExternalSigner(endpoint string) (*ExternalSigner, error) {
+	client, err := rpc.Dial(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	extsigner := &ExternalSigner{
+		client:   client,
+		endpoint: endpoint,
+	}
+	// Check if reachable
+	version, err := extsigner.pingVersion()
+	if err != nil {
+		return nil, err
+	}
+	extsigner.status = fmt.Sprintf("ok [version=%v]", version)
+	return extsigner, nil
+}
+
+func (api *ExternalSigner) URL() accounts.URL {
+	return accounts.URL{
+		Scheme: "extapi",
+		Path:   api.endpoint,
+	}
+}
+
+func (api *ExternalSigner) Status() (string, error) {
+	return api.status, nil
+}
+
+func (api *ExternalSigner) Open(passphrase string) error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Close() error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Accounts() []accounts.Account {
+	var accnts []accounts.Account
+	res, err := api.listAccounts()
+	if err != nil {
+		log.Error("account listing failed", "error", err)
+		return accnts
+	}
+	for _, addr := range res {
+		accnts = append(accnts, accounts.Account{
+			URL: accounts.URL{
+				Scheme: "extapi",
+				Path:   api.endpoint,
+			},
+			Address: addr,
+		})
+	}
+	api.cacheMu.Lock()
+	api.cache = accnts
+	api.cacheMu.Unlock()
+	return accnts
+}
+
+func (api *ExternalSigner) Contains(account accounts.Account) bool {
+	api.cacheMu.RLock()
+	defer api.cacheMu.RUnlock()
+	for _, a := range api.cache {
+		if a.Address == account.Address && (account.URL == (accounts.URL{}) || account.URL == api.URL()) {
+			return true
+		}
+	}
+	return false
+}
+
+func (api *ExternalSigner) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	log.Error("operation SelfDerive not supported on external signers")
+}
+
+func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("operation not supported on external signers")
+}
+
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		mimeType,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(data)); err != nil {
+		return nil, err
+	}
+	// If V is on 27/28-form, convert to to 0/1 for Clique
+	if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
+		res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		accounts.MimetypeTextPlain,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(text)); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	res := ethapi.SignTransactionResult{}
+	to := common.NewMixedcaseAddress(*tx.To())
+	data := hexutil.Bytes(tx.Data())
+	args := &core.SendTxArgs{
+		Data:     &data,
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		Value:    hexutil.Big(*tx.Value()),
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: hexutil.Big(*tx.GasPrice()),
+		To:       &to,
+		From:     common.NewMixedcaseAddress(account.Address),
+	}
+
+	if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
+		return nil, err
+	}
+	return res.Tx, nil
+}
+
+func (api *ExternalSigner) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+
+func (api *ExternalSigner) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+func (api *ExternalSigner) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+
+func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
+	var res []common.Address
+	if err := api.client.Call(&res, "account_list"); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) pingVersion() (string, error) {
+	var v string
+	if err := api.client.Call(&v, "account_version"); err != nil {
+		return "", err
+	}
+	return v, nil
+}

accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
-// at m/44'/60'/0'/1, etc.
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

accounts/keystore/account_cache.go

@@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error {
 		case (addr == common.Address{}):
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
-			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+			return &accounts.Account{
+				Address: addr,
+				URL:     accounts.URL{Scheme: KeyStoreScheme, Path: path},
+			}
 		}
 		return nil
 	}

accounts/keystore/key.go

@@ -66,19 +66,19 @@ type plainKeyJSON struct {
 
 type encryptedKeyJSONV3 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version int        `json:"version"`
 }
 
 type encryptedKeyJSONV1 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version string     `json:"version"`
 }
 
-type cryptoJSON struct {
+type CryptoJSON struct {
 	Cipher       string                 `json:"cipher"`
 	CipherText   string                 `json:"ciphertext"`
 	CipherParams cipherparamsJSON       `json:"cipherparams"`
@@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	if err != nil {
 		return nil, accounts.Account{}, err
 	}
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
+	}
 	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
@@ -179,26 +182,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	return key, a, err
 }
 
-func writeKeyFile(file string, content []byte) error {
+func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	// Create the keystore directory with appropriate permissions
 	// in case it is not present yet.
 	const dirPerm = 0700
 	if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
-		return err
+		return "", err
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
 	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err := f.Write(content); err != nil {
 		f.Close()
 		os.Remove(f.Name())
-		return err
+		return "", err
 	}
 	f.Close()
-	return os.Rename(f.Name(), file)
+	return f.Name(), nil
+}
+
+func writeKeyFile(file string, content []byte) error {
+	name, err := writeTemporaryKeyFile(file, content)
+	if err != nil {
+		return err
+	}
+	return os.Rename(name, file)
 }
 
 // keyFileName implements the naming convention for keyfiles:
@@ -216,5 +227,6 @@ func toISO8601(t time.Time) string {
 	} else {
 		tz = fmt.Sprintf("%03d00", offset/3600)
 	}
-	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
+	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
+		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
 }

accounts/keystore/keystore.go

@@ -78,7 +78,7 @@ type unlocked struct {
 // NewKeyStore creates a keystore for the given directory.
 func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
 	ks.init(keydir)
 	return ks
 }
@@ -137,8 +137,10 @@ func (ks *KeyStore) refreshWallets() {
 	accs := ks.cache.accounts()
 
 	// Transform the current list of wallets into the new one
-	wallets := make([]accounts.Wallet, 0, len(accs))
-	events := []accounts.WalletEvent{}
+	var (
+		wallets = make([]accounts.Wallet, 0, len(accs))
+		events  []accounts.WalletEvent
+	)
 
 	for _, account := range accs {
 		// Drop wallets while they were in front of the next account

accounts/keystore/passphrase.go

@@ -35,6 +35,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -72,6 +73,10 @@ type keyStorePassphrase struct {
 	keysDirPath string
 	scryptN     int
 	scryptP     int
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
 }
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 
 // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
 func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
-	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, rand.Reader, auth)
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
 	return a.Address, err
 }
 
@@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
 	if err != nil {
 		return err
 	}
-	return writeKeyFile(filename, keyjson)
+	// Write into temporary file
+	tmpName, err := writeTemporaryKeyFile(filename, keyjson)
+	if err != nil {
+		return err
+	}
+	if !ks.skipKeyFileVerification {
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.Address, tmpName, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted file is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/ethereum/go-ethereum/issues." +
+				"The error was : %s"
+			return fmt.Errorf(msg, tmpName, err)
+		}
+	}
+	return os.Rename(tmpName, filename)
 }
 
 func (ks keyStorePassphrase) JoinPath(filename string) string {
@@ -112,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	return filepath.Join(ks.keysDirPath, filename)
 }
 
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
-// blob that can be decrypted later on.
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
-	authArray := []byte(auth)
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
 
 	salt := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := make([]byte, aes.BlockSize) // 16
 	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
 
@@ -144,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 	scryptParamsJSON["p"] = scryptP
 	scryptParamsJSON["dklen"] = scryptDKLen
 	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-
 	cipherParamsJSON := cipherparamsJSON{
 		IV: hex.EncodeToString(iv),
 	}
 
-	cryptoStruct := cryptoJSON{
+	cryptoStruct := CryptoJSON{
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
@@ -157,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
 	encryptedKeyJSONV3 := encryptedKeyJSONV3{
 		hex.EncodeToString(key.Address[:]),
 		cryptoStruct,
@@ -204,42 +234,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	}, nil
 }
 
-func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
-	if keyProtected.Version != version {
-		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
+	if cryptoJson.Cipher != "aes-128-ctr" {
+		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
 	}
-
-	if keyProtected.Crypto.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
-	}
-
-	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
+	mac, err := hex.DecodeString(cryptoJson.MAC)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
+	derivedKey, err := getKDFKey(cryptoJson, auth)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
 	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
+		return nil, ErrDecrypt
 	}
 
 	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
+
+func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
+	if keyProtected.Version != version {
+		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+	}
+	keyId = uuid.Parse(keyProtected.Id)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -280,7 +316,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
 	return plainText, keyId, err
 }
 
-func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 	authArray := []byte(auth)
 	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
 	if err != nil {

accounts/keystore/plain_test.go

@@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 		t.Fatal(err)
 	}
 	if encrypted {
-		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
+		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
 		ks = &keyStorePlain{d}
 	}
@@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
 
 func TestV1_2(t *testing.T) {
 	t.Parallel()
-	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
+	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
 	addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
 	file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
 	k, err := ks.GetKey(addr, file, "g")

accounts/keystore/presale.go

@@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
 		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL: accounts.URL{
+			Scheme: KeyStoreScheme,
+			Path:   keyStore.JoinPath(keyFileName(key.Address)),
+		},
+	}
 	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }

accounts/keystore/wallet.go

@@ -22,6 +22,7 @@ import (
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 )
 
 // keystoreWallet implements the accounts.Wallet interface for the original
@@ -52,8 +53,8 @@ func (w *keystoreWallet) Status() (string, error) {
 // is no connection or decryption step necessary to access the list of accounts.
 func (w *keystoreWallet) Open(passphrase string) error { return nil }
 
-// Close implements accounts.Wallet, but is a noop for plain wallets since is no
-// meaningful open operation.
+// Close implements accounts.Wallet, but is a noop for plain wallets since there
+// is no meaningful open operation.
 func (w *keystoreWallet) Close() error { return nil }
 
 // Accounts implements accounts.Wallet, returning an account list consisting of
@@ -78,60 +79,67 @@ func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (account
 // there is no notion of hierarchical account derivation for plain keystore accounts.
 func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
 
-// SignHash implements accounts.Wallet, attempting to sign the given hash with
+// signHash attempts to sign the given hash with
 // the given account. If the wallet does not wrap this particular account, an
 // error is returned to avoid account leakage (even though in theory we may be
 // able to sign via our shared keystore backend).
-func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *keystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
-	if account.Address != w.account.Address {
-		return nil, accounts.ErrUnknownAccount
-	}
-	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
 	}
 	// Account seems valid, request the keystore to sign
 	return w.keystore.SignHash(account, hash)
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, crypto.Keccak256(data))
+}
+
+func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
+// SignTextWithPassphrase implements accounts.Wallet, attempting to sign the
+// given hash with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet, attempting to sign the given transaction
 // with the given account. If the wallet does not wrap this particular account,
 // an error is returned to avoid account leakage (even though in theory we may
 // be able to sign via our shared keystore backend).
 func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
 	// Make sure the requested account is contained within
-	if account.Address != w.account.Address {
-		return nil, accounts.ErrUnknownAccount
-	}
-	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
 	}
 	// Account seems valid, request the keystore to sign
 	return w.keystore.SignTx(account, tx, chainID)
 }
 
-// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
-// given hash with the given account using passphrase as extra authentication.
-func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
-	// Make sure the requested account is contained within
-	if account.Address != w.account.Address {
-		return nil, accounts.ErrUnknownAccount
-	}
-	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
-		return nil, accounts.ErrUnknownAccount
-	}
-	// Account seems valid, request the keystore to sign
-	return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
-}
-
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
 // transaction with the given account using passphrase as extra authentication.
 func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
 	// Make sure the requested account is contained within
-	if account.Address != w.account.Address {
-		return nil, accounts.ErrUnknownAccount
-	}
-	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
 	}
 	// Account seems valid, request the keystore to sign

accounts/usbwallet/hub.go

@@ -150,8 +150,10 @@ func (hub *Hub) refreshWallets() {
 	// Transform the current list of wallets into the new one
 	hub.stateLock.Lock()
 
-	wallets := make([]accounts.Wallet, 0, len(devices))
-	events := []accounts.WalletEvent{}
+	var (
+		wallets = make([]accounts.Wallet, 0, len(devices))
+		events  []accounts.WalletEvent
+	)
 
 	for _, device := range devices {
 		url := accounts.URL{Scheme: hub.scheme, Path: device.Path}

accounts/usbwallet/ledger.go

@@ -257,7 +257,9 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er
 
 	// Decode the hex sting into an Ethereum address and return
 	var address common.Address
-	hex.Decode(address[:], hexstr)
+	if _, err = hex.Decode(address[:], hexstr); err != nil {
+		return common.Address{}, err
+	}
 	return address, nil
 }
 
@@ -350,7 +352,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/trezor.go

@@ -28,7 +28,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/accounts"
-	"github.com/ethereum/go-ethereum/accounts/usbwallet/internal/trezor"
+	"github.com/ethereum/go-ethereum/accounts/usbwallet/trezor"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -41,6 +41,9 @@ import (
 // encoded passphrase.
 var ErrTrezorPINNeeded = errors.New("trezor: pin needed")
 
+// ErrTrezorPassphraseNeeded is returned if opening the trezor requires a passphrase
+var ErrTrezorPassphraseNeeded = errors.New("trezor: passphrase needed")
+
 // errTrezorReplyInvalidHeader is the error message returned by a Trezor data exchange
 // if the device replies with a mismatching header. This usually means the device
 // is in browser mode.
@@ -48,12 +51,13 @@ var errTrezorReplyInvalidHeader = errors.New("trezor: invalid reply header")
 
 // trezorDriver implements the communication with a Trezor hardware wallet.
 type trezorDriver struct {
-	device  io.ReadWriter // USB device connection to communicate through
-	version [3]uint32     // Current version of the Trezor firmware
-	label   string        // Current textual label of the Trezor device
-	pinwait bool          // Flags whether the device is waiting for PIN entry
-	failure error         // Any failure that would make the device unusable
-	log     log.Logger    // Contextual logger to tag the trezor with its id
+	device         io.ReadWriter // USB device connection to communicate through
+	version        [3]uint32     // Current version of the Trezor firmware
+	label          string        // Current textual label of the Trezor device
+	pinwait        bool          // Flags whether the device is waiting for PIN entry
+	passphrasewait bool          // Flags whether the device is waiting for passphrase entry
+	failure        error         // Any failure that would make the device unusable
+	log            log.Logger    // Contextual logger to tag the trezor with its id
 }
 
 // newTrezorDriver creates a new instance of a Trezor USB protocol driver.
@@ -79,7 +83,7 @@ func (w *trezorDriver) Status() (string, error) {
 }
 
 // Open implements usbwallet.driver, attempting to initialize the connection to
-// the Trezor hardware wallet. Initializing the Trezor is a two phase operation:
+// the Trezor hardware wallet. Initializing the Trezor is a two or three phase operation:
 //  * The first phase is to initialize the connection and read the wallet's
 //    features. This phase is invoked is the provided passphrase is empty. The
 //    device will display the pinpad as a result and will return an appropriate
@@ -87,11 +91,13 @@ func (w *trezorDriver) Status() (string, error) {
 //  * The second phase is to unlock access to the Trezor, which is done by the
 //    user actually providing a passphrase mapping a keyboard keypad to the pin
 //    number of the user (shuffled according to the pinpad displayed).
+//  * If needed the device will ask for passphrase which will require calling
+//    open again with the actual passphrase (3rd phase)
 func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error {
 	w.device, w.failure = device, nil
 
 	// If phase 1 is requested, init the connection and wait for user callback
-	if passphrase == "" {
+	if passphrase == "" && !w.passphrasewait {
 		// If we're already waiting for a PIN entry, insta-return
 		if w.pinwait {
 			return ErrTrezorPINNeeded
@@ -104,26 +110,46 @@ func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error {
 		w.version = [3]uint32{features.GetMajorVersion(), features.GetMinorVersion(), features.GetPatchVersion()}
 		w.label = features.GetLabel()
 
-		// Do a manual ping, forcing the device to ask for its PIN
+		// Do a manual ping, forcing the device to ask for its PIN and Passphrase
 		askPin := true
-		res, err := w.trezorExchange(&trezor.Ping{PinProtection: &askPin}, new(trezor.PinMatrixRequest), new(trezor.Success))
+		askPassphrase := true
+		res, err := w.trezorExchange(&trezor.Ping{PinProtection: &askPin, PassphraseProtection: &askPassphrase}, new(trezor.PinMatrixRequest), new(trezor.PassphraseRequest), new(trezor.Success))
 		if err != nil {
 			return err
 		}
 		// Only return the PIN request if the device wasn't unlocked until now
-		if res == 1 {
-			return nil // Device responded with trezor.Success
+		switch res {
+		case 0:
+			w.pinwait = true
+			return ErrTrezorPINNeeded
+		case 1:
+			w.pinwait = false
+			w.passphrasewait = true
+			return ErrTrezorPassphraseNeeded
+		case 2:
+			return nil // responded with trezor.Success
 		}
-		w.pinwait = true
-		return ErrTrezorPINNeeded
 	}
 	// Phase 2 requested with actual PIN entry
-	w.pinwait = false
-
-	if _, err := w.trezorExchange(&trezor.PinMatrixAck{Pin: &passphrase}, new(trezor.Success)); err != nil {
-		w.failure = err
-		return err
+	if w.pinwait {
+		w.pinwait = false
+		res, err := w.trezorExchange(&trezor.PinMatrixAck{Pin: &passphrase}, new(trezor.Success), new(trezor.PassphraseRequest))
+		if err != nil {
+			w.failure = err
+			return err
+		}
+		if res == 1 {
+			w.passphrasewait = true
+			return ErrTrezorPassphraseNeeded
+		}
+	} else if w.passphrasewait {
+		w.passphrasewait = false
+		if _, err := w.trezorExchange(&trezor.PassphraseAck{Passphrase: &passphrase}, new(trezor.Success)); err != nil {
+			w.failure = err
+			return err
+		}
 	}
+
 	return nil
 }
 
@@ -221,7 +247,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

accounts/usbwallet/wallet.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/karalabe/hid"
 )
@@ -273,9 +274,7 @@ func (w *wallet) close() error {
 	w.device = nil
 
 	w.accounts, w.paths = nil, nil
-	w.driver.Close()
-
-	return nil
+	return w.driver.Close()
 }
 
 // Accounts implements accounts.Wallet, returning the list of accounts pinned to
@@ -495,12 +494,28 @@ func (w *wallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainSt
 	w.deriveChain = chain
 }
 
-// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// signHash implements accounts.Wallet, however signing arbitrary data is not
 // supported for hardware wallets, so this method will always return an error.
-func (w *wallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	return nil, accounts.ErrNotSupported
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+// SignDataWithPassphrase implements accounts.Wallet, attempting to sign the given
+// data with the given account using passphrase as extra authentication.
+// Since USB wallets don't rely on passphrases, these are silently ignored.
+func (w *wallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	return w.SignData(account, mimeType, data)
+}
+
+func (w *wallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
 // wallet to request a confirmation from the user. It returns either the signed
 // transaction or a failure if the user denied the transaction.
@@ -550,8 +565,8 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID
 // SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
 // data is not supported for Ledger wallets, so this method will always return
 // an error.
-func (w *wallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
-	return w.SignHash(account, hash)
+func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	return w.SignText(account, accounts.TextHash(text))
 }
 
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.3.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://dl.google.com/go/go1.12.windows-%GETH_ARCH%.zip
+  - 7z x go1.12.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci-notes.md

@@ -7,27 +7,34 @@ Canonical.
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
 
+## Launchpad
+
 The packages are built and served by launchpad.net. We generate a Debian source package
 for each distribution and upload it. Their builder picks up the source package, builds it
 and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
-variable which Travis CI makes available to certain builds.
+by a team member for source package uploads.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.10, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.10 and Debian packaging tools:
+Add the gophers PPA and install Go 1.11 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -80,6 +80,7 @@ var (
 		executablePath("puppeth"),
 		executablePath("rlpdump"),
 		executablePath("wnode"),
+		executablePath("clef"),
 	}
 
 	// Files that end up in the swarm*.zip archive.
@@ -118,6 +119,10 @@ var (
 			BinaryName:  "wnode",
 			Description: "Ethereum Whisper diagnostic tool",
 		},
+		{
+			BinaryName:  "clef",
+			Description: "Ethereum account management tool.",
+		},
 	}
 
 	// A debian package is created for all executables listed here.
@@ -156,7 +161,7 @@ var (
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
 	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
 	// Note: artful is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
+	debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -320,9 +325,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	var (
-		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
-	)
+	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -332,14 +335,11 @@ func doTest(cmdline []string) {
 	}
 	packages = build.ExpandPackagesNoVendor(packages)
 
-	// Run analysis tools before the tests.
-	build.MustRun(goTool("vet", packages...))
-
 	// Run the actual tests.
-	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest.Args = append(gotest.Args, "-p", "1")
+	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -446,11 +446,8 @@ func archiveBasename(arch string, archiveVersion string) string {
 func archiveUpload(archive string, blobstore string, signer string) error {
 	// If signing was requested, generate the signature files
 	if signer != "" {
-		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
-		if err != nil {
-			return fmt.Errorf("invalid base64 %s", signer)
-		}
-		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
+		key := getenvBase64(signer)
+		if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
 			return err
 		}
 	}
@@ -493,7 +490,8 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		upload  = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
 		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
 		now     = time.Now()
 	)
@@ -503,11 +501,7 @@ func doDebianSource(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Import the signing key.
-	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatal("invalid base64 PPA_SIGNING_KEY")
-		}
+	if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
@@ -518,22 +512,58 @@ func doDebianSource(cmdline []string) {
 		for _, distro := range debDistros {
 			meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
-			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
+			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
 			build.MustRun(debuild)
 
-			changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
-			changes = filepath.Join(*workdir, changes)
+			var (
+				basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
+				source   = filepath.Join(*workdir, basename+".tar.xz")
+				dsc      = filepath.Join(*workdir, basename+".dsc")
+				changes  = filepath.Join(*workdir, basename+"_source.changes")
+			)
 			if *signer != "" {
 				build.MustRunCommand("debsign", changes)
 			}
 			if *upload != "" {
-				build.MustRunCommand("dput", *upload, changes)
+				ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes})
 			}
 		}
 	}
 }
 
+func ppaUpload(workdir, ppa, sshUser string, files []string) {
+	p := strings.Split(ppa, "/")
+	if len(p) != 2 {
+		log.Fatal("-upload PPA name must contain single /")
+	}
+	if sshUser == "" {
+		sshUser = p[0]
+	}
+	incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
+	// Create the SSH identity file if it doesn't exist.
+	var idfile string
+	if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+		idfile = filepath.Join(workdir, "sshkey")
+		if _, err := os.Stat(idfile); os.IsNotExist(err) {
+			ioutil.WriteFile(idfile, sshkey, 0600)
+		}
+	}
+	// Upload
+	dest := sshUser + "@ppa.launchpad.net"
+	if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getenvBase64(variable string) []byte {
+	dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+	if err != nil {
+		log.Fatal("invalid base64 " + variable)
+	}
+	return []byte(dec)
+}
+
 func makeWorkdir(wdflag string) string {
 	var err error
 	if wdflag != "" {
@@ -775,12 +805,8 @@ func doAndroidArchive(cmdline []string) {
 	if os.Getenv("ANDROID_HOME") == "" {
 		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
 	}
-	if os.Getenv("ANDROID_NDK") == "" {
-		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
-	}
 	// Build the Android archive and Maven resources
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
-	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {
@@ -805,15 +831,10 @@ func doAndroidArchive(cmdline []string) {
 	os.Rename(archive, meta.Package+".aar")
 	if *signer != "" && *deploy != "" {
 		// Import the signing key into the local GPG instance
-		b64key := os.Getenv(*signer)
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatalf("invalid base64 %s", *signer)
-		}
+		key := getenvBase64(*signer)
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
-
 		keyID, err := build.PGPKeyID(string(key))
 		if err != nil {
 			log.Fatal(err)
@@ -1040,7 +1061,7 @@ func xgoTool(args []string) *exec.Cmd {
 func doPurge(cmdline []string) {
 	var (
 		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
-		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
 	)
 	flag.CommandLine.Parse(cmdline)
 

build/deb/ethereum-swarm/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.10
+Build-Depends: debhelper (>= 8.0.0), golang-1.11
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb/ethereum-swarm/deb.rules

@@ -4,8 +4,11 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/deb/ethereum/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.10
+Build-Depends: debhelper (>= 8.0.0), golang-1.11
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb/ethereum/deb.rules

@@ -4,8 +4,11 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/travis_keepalive.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# travis_keepalive runs the given command and preserves its return value,
+# while it forks a child process what periodically produces a log line,
+# so that Travis won't abort the build after 10 minutes.
+
+# Why?
+# `t.Log()` in Go holds the buffer until the test does not pass or fail,
+# and `-race` can increase the execution time by 2-20x.
+
+set -euo pipefail
+
+readonly KEEPALIVE_INTERVAL=300 # seconds => 5m
+
+main() {
+  keepalive
+  $@
+}
+
+# Keepalive produces a log line in each KEEPALIVE_INTERVAL.
+keepalive() {
+  local child_pid
+  # Note: We fork here!
+  repeat "keepalive" &
+  child_pid=$!
+  ensureChildOnEXIT "${child_pid}"
+}
+
+repeat() {
+  local this="$1"
+  while true; do
+    echo "${this}"
+    sleep "${KEEPALIVE_INTERVAL}"
+  done
+}
+
+# Ensures that the child gets killed on normal program exit.
+ensureChildOnEXIT() {
+  # Note: SIGINT and SIGTERM are forwarded to the child process by Bash
+  # automatically, so we don't have to deal with signals.
+
+  local child_pid="$1"
+  trap "kill ${child_pid}" EXIT
+}
+
+main "$@"

build/update-license.go

@@ -1,3 +1,19 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 // +build none
 
 /*

cmd/abigen/main.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -37,6 +37,9 @@ var (
 	solcFlag = flag.String("solc", "solc", "Solidity compiler to use if source builds are requested")
 	excFlag  = flag.String("exc", "", "Comma separated types to exclude from binding")
 
+	vyFlag    = flag.String("vy", "", "Path to the Ethereum contract Vyper source to build and bind")
+	vyperFlag = flag.String("vyper", "vyper", "Vyper compiler to use if source builds are requested")
+
 	pkgFlag  = flag.String("pkg", "", "Package name to generate the binding into")
 	outFlag  = flag.String("out", "", "Output file for the generated binding (default = stdout)")
 	langFlag = flag.String("lang", "go", "Destination language for the bindings (go, java, objc)")
@@ -46,11 +49,14 @@ func main() {
 	// Parse and ensure all needed inputs are specified
 	flag.Parse()
 
-	if *abiFlag == "" && *solFlag == "" {
-		fmt.Printf("No contract ABI (--abi) or Solidity source (--sol) specified\n")
+	if *abiFlag == "" && *solFlag == "" && *vyFlag == "" {
+		fmt.Printf("No contract ABI (--abi), Solidity source (--sol), or Vyper source (--vy) specified\n")
 		os.Exit(-1)
-	} else if (*abiFlag != "" || *binFlag != "" || *typFlag != "") && *solFlag != "" {
-		fmt.Printf("Contract ABI (--abi), bytecode (--bin) and type (--type) flags are mutually exclusive with the Solidity source (--sol) flag\n")
+	} else if (*abiFlag != "" || *binFlag != "" || *typFlag != "") && (*solFlag != "" || *vyFlag != "") {
+		fmt.Printf("Contract ABI (--abi), bytecode (--bin) and type (--type) flags are mutually exclusive with the Solidity (--sol) and Vyper (--vy) flags\n")
+		os.Exit(-1)
+	} else if *solFlag != "" && *vyFlag != "" {
+		fmt.Printf("Solidity (--sol) and Vyper (--vy) flags are mutually exclusive\n")
 		os.Exit(-1)
 	}
 	if *pkgFlag == "" {
@@ -75,7 +81,7 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" || *abiFlag == "-" {
+	if *solFlag != "" || *vyFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
@@ -84,13 +90,21 @@ func main() {
 
 		var contracts map[string]*compiler.Contract
 		var err error
-		if *solFlag != "" {
+
+		switch {
+		case *solFlag != "":
 			contracts, err = compiler.CompileSolidity(*solcFlag, *solFlag)
 			if err != nil {
 				fmt.Printf("Failed to build Solidity contract: %v\n", err)
 				os.Exit(-1)
 			}
-		} else {
+		case *vyFlag != "":
+			contracts, err = compiler.CompileVyper(*vyperFlag, *vyFlag)
+			if err != nil {
+				fmt.Printf("Failed to build Vyper contract: %v\n", err)
+				os.Exit(-1)
+			}
+		default:
 			contracts, err = contractsFromStdin()
 			if err != nil {
 				fmt.Printf("Failed to read input ABIs from STDIN: %v\n", err)
@@ -102,7 +116,11 @@ func main() {
 			if exclude[strings.ToLower(name)] {
 				continue
 			}
-			abi, _ := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
+			abi, err := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
+			if err != nil {
+				fmt.Printf("Failed to parse ABIs from compiler output: %v\n", err)
+				os.Exit(-1)
+			}
 			abis = append(abis, string(abi))
 			bins = append(bins, contract.Code)
 
@@ -111,14 +129,20 @@ func main() {
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
+		var abi []byte
+		var err error
+		if *abiFlag == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(*abiFlag)
+		}
 		if err != nil {
 			fmt.Printf("Failed to read input ABI: %v\n", err)
 			os.Exit(-1)
 		}
 		abis = append(abis, string(abi))
 
-		bin := []byte{}
+		var bin []byte
 		if *binFlag != "" {
 			if bin, err = ioutil.ReadFile(*binFlag); err != nil {
 				fmt.Printf("Failed to read input bytecode: %v\n", err)
@@ -155,6 +179,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

cmd/bootnode/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
@@ -37,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -85,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -111,26 +112,38 @@ func main() {
 		if !realaddr.IP.IsLoopback() {
 			go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
 		}
-		// TODO: react to external IP changes over time.
 		if ext, err := natm.ExternalIP(); err == nil {
 			realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
 		}
 	}
 
+	printNotice(&nodeKey.PublicKey, *realaddr)
+
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
-			PrivateKey:   nodeKey,
-			AnnounceAddr: realaddr,
-			NetRestrict:  restrictList,
+			PrivateKey:  nodeKey,
+			NetRestrict: restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}
 
 	select {}
 }
+
+func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) {
+	if addr.IP.IsUnspecified() {
+		addr.IP = net.IP{127, 0, 0, 1}
+	}
+	n := enode.NewV4(nodeKey, addr.IP, 0, addr.Port)
+	fmt.Println(n.String())
+	fmt.Println("Note: you're using cmd/bootnode, a developer tool.")
+	fmt.Println("We recommend using a regular node as bootstrap node for production deployments.")
+}

cmd/clef/README.md

@@ -16,7 +16,8 @@ Check out
 
 * the [tutorial](tutorial.md) for some concrete examples on how the signer works.
 * the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. 
-
+* the [data types](datatypes.md) for detailed information on the json types used in the communication between
+  clef and an external UI 
 
 ## Command line flags
 Clef accepts the following command line options:
@@ -24,28 +25,34 @@ Clef accepts the following command line options:
 COMMANDS:
    init    Initialize the signer, generate secret storage
    attest  Attest that a js-file is to be used
-   addpw   Store a credential for a keystore file
+   setpw   Store a credential for a keystore file
+   gendoc  Generate documentation about json-rpc format
    help    Shows a list of commands or help for one command
-
+   
 GLOBAL OPTIONS:
    --loglevel value        log level to emit to the screen (default: 4)
    --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
-   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
-   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
+   --chainid value         Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
    --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
    --nousb                 Disables monitoring for and managing USB hardware wallets
    --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcvhosts value       Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
+   --ipcdisable            Disable the IPC-RPC server
+   --ipcpath               Filename for IPC socket/pipe within the datadir (explicit paths escape it)
+   --rpc                   Enable the HTTP-RPC server
    --rpcport value         HTTP-RPC server listening port (default: 8550)
-   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --signersecret value    A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
    --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
    --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
    --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
    --rules value           Enable rule-engine (default: "rules.json")
-   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
-   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
+   --stdio-ui-test         Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
+   --advanced              If enabled, issues warnings instead of rejections for suspicious requests. Default off
    --help, -h              show help
    --version, -v           print the version
-
+   
 ```
 
 
@@ -91,7 +98,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +136,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -189,7 +196,9 @@ None
   "method": "account_new",
   "params": []
 }
-
+```
+Response
+```
 {
   "id": 0,
   "jsonrpc": "2.0",
@@ -222,7 +231,9 @@ None
   "jsonrpc": "2.0",
   "method": "account_list"
 }
-
+```
+Response
+```
 {
   "id": 1,
   "jsonrpc": "2.0",
@@ -285,8 +296,8 @@ Response
 
 ```json
 {
+  "id": 2,
   "jsonrpc": "2.0",
-  "id": 67,
   "error": {
     "code": -32000,
     "message": "Request denied"
@@ -298,6 +309,7 @@ Response
 
 ```json
 {
+  "id": 67,
   "jsonrpc": "2.0",
   "method": "account_signTransaction",
   "params": [
@@ -311,8 +323,7 @@ Response
       "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
     },
     "safeSend(address)"
-  ],
-  "id": 67
+  ]
 }
 ```
 Response
@@ -346,15 +357,18 @@ Bash example:
 {"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
 ```
 
-
-### account_sign
+### account_signData
 
 #### Sign data
    Signs a chunk of data and returns the calculated signature.
 
 #### Arguments
+  - content type [string]: type of signed data
+     - `text/validator`: hex data with custom validator defined in a contract
+     - `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
+     - `text/plain`: simple hex data validated by `account_ecRecover`
   - account [address]: account to sign with
-  - data [data]: data to sign
+  - data [object]: data to sign
 
 #### Result
   - calculated signature [data]
@@ -364,8 +378,9 @@ Bash example:
 {
   "id": 3,
   "jsonrpc": "2.0",
-  "method": "account_sign",
+  "method": "account_signData",
   "params": [
+    "data/plain",
     "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
     "0xaabbccdd"
   ]
@@ -381,11 +396,109 @@ Response
 }
 ```
 
+### account_signTypedData
+
+#### Sign data
+   Signs a chunk of structured data conformant to [EIP712]([EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md)) and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [object]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 68,
+  "jsonrpc": "2.0",
+  "method": "account_signTypedData",
+  "params": [
+    "0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826",
+    {
+      "types": {
+        "EIP712Domain": [
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "version",
+            "type": "string"
+          },
+          {
+            "name": "chainId",
+            "type": "uint256"
+          },
+          {
+            "name": "verifyingContract",
+            "type": "address"
+          }
+        ],
+        "Person": [
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "wallet",
+            "type": "address"
+          }
+        ],
+        "Mail": [
+          {
+            "name": "from",
+            "type": "Person"
+          },
+          {
+            "name": "to",
+            "type": "Person"
+          },
+          {
+            "name": "contents",
+            "type": "string"
+          }
+        ]
+      },
+      "primaryType": "Mail",
+      "domain": {
+        "name": "Ether Mail",
+        "version": "1",
+        "chainId": 1,
+        "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC"
+      },
+      "message": {
+        "from": {
+          "name": "Cow",
+          "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826"
+        },
+        "to": {
+          "name": "Bob",
+          "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB"
+        },
+        "contents": "Hello, Bob!"
+      }
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+    "id": 1,
+    "jsonrpc": "2.0",
+    "result": "0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c"
+}
+```
+
 ### account_ecRecover
 
-#### Recover address
-   Derive the address from the account that was used to sign data from the data and signature.
-   
+#### Sign data
+
+Derive the address from the account that was used to sign data with content type `text/plain` and the signature.
+
 #### Arguments
   - data [data]: data that was signed
   - signature [data]: the signature to verify
@@ -400,6 +513,7 @@ Response
   "jsonrpc": "2.0",
   "method": "account_ecRecover",
   "params": [
+    "data/plain",
     "0xaabbccdd",
     "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
   ]
@@ -413,7 +527,6 @@ Response
   "jsonrpc": "2.0",
   "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
 }
-
 ```
 
 ### account_import
@@ -458,7 +571,7 @@ Response
       },
       "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
       "version": 3
-    },
+    }
   ]
 }
 ```
@@ -548,7 +661,7 @@ OBS! A slight deviation from `json` standard is in place: every request and resp
 Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
 things simpler for both parties.
 
-### ApproveTx
+### ApproveTx / `ui_approveTx`
 
 Invoked when there's a transaction for approval.
 
@@ -560,13 +673,13 @@ Here's a method invocation:
 
 curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
 ```
-
+Results in the following invocation on the UI:
 ```json
 
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -611,7 +724,7 @@ curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","me
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -654,7 +767,7 @@ One which has missing `to`, but with no `data`:
 {
   "jsonrpc": "2.0",
   "id": 3,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -683,33 +796,7 @@ One which has missing `to`, but with no `data`:
 }
 ```
 
-### ApproveExport
-
-Invoked when a request to export an account has been made.
-
-#### Sample call
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 7,
-  "method": "ApproveExport",
-  "params": [
-    {
-      "address": "0x0000000000000000000000000000000000000000",
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-
-```
-
-### ApproveListing
+### ApproveListing / `ui_approveListing`
 
 Invoked when a request for account listing has been made.
 
@@ -720,7 +807,7 @@ Invoked when a request for account listing has been made.
 {
   "jsonrpc": "2.0",
   "id": 5,
-  "method": "ApproveListing",
+  "method": "ui_approveListing",
   "params": [
     {
       "accounts": [
@@ -747,7 +834,7 @@ Invoked when a request for account listing has been made.
 ```
 
 
-### ApproveSignData
+### ApproveSignData / `ui_approveSignData`
 
 #### Sample call
 
@@ -755,7 +842,7 @@ Invoked when a request for account listing has been made.
 {
   "jsonrpc": "2.0",
   "id": 4,
-  "method": "ApproveSignData",
+  "method": "ui_approveSignData",
   "params": [
     {
       "address": "0x123409812340981234098123409812deadbeef42",
@@ -773,7 +860,7 @@ Invoked when a request for account listing has been made.
 
 ```
 
-### ShowInfo
+### ShowInfo / `ui_showInfo`
 
 The UI should show the info to the user. Does not expect response.
 
@@ -783,7 +870,7 @@ The UI should show the info to the user. Does not expect response.
 {
   "jsonrpc": "2.0",
   "id": 9,
-  "method": "ShowInfo",
+  "method": "ui_showInfo",
   "params": [
     {
       "text": "Tests completed"
@@ -793,7 +880,7 @@ The UI should show the info to the user. Does not expect response.
 
 ```
 
-### ShowError
+### ShowError / `ui_showError`
 
 The UI should show the info to the user. Does not expect response.
 
@@ -812,7 +899,7 @@ The UI should show the info to the user. Does not expect response.
 
 ```
 
-### OnApproved
+### OnApprovedTx / `ui_onApprovedTx`
 
 `OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
 
@@ -820,7 +907,7 @@ When implementing rate-limited rules, this callback should be used.
 
 TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
 
-### OnSignerStartup
+### OnSignerStartup / `ui_onSignerStartup`
 
 This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
 in k/v-form.
@@ -831,7 +918,7 @@ Example call:
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "OnSignerStartup",
+  "method": "ui_onSignerStartup",
   "params": [
     {
       "info": {
@@ -862,7 +949,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 
@@ -875,3 +962,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/clef/datatypes.md

@@ -0,0 +1,224 @@
+## UI Client interface
+
+These data types are defined in the channel between clef and the UI
+### SignDataRequest
+
+SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message`
+
+Example:
+```json
+{
+  "content_type": "text/plain",
+  "address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+  "raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
+  "message": [
+    {
+      "name": "message",
+      "value": "\u0019Ethereum Signed Message:\n11hello world",
+      "type": "text/plain"
+    }
+  ],
+  "hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignDataResponse - approve
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": true
+}
+```
+### SignDataResponse - deny
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": false
+}
+```
+### SignTxRequest
+
+SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
+
+As in any request, it's important to consider that the `meta` info also contains untrusted data.
+
+The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x1",
+    "data": "0x01020304"
+  },
+  "call_info": [
+    {
+      "type": "Warning",
+      "message": "Something looks odd, show this message as a warning"
+    },
+    {
+      "type": "Info",
+      "message": "User should see this aswell"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignTxResponse - approve
+
+Response to request to sign a transaction. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x4",
+    "data": "0x04030201"
+  },
+  "approved": true
+}
+```
+### SignTxResponse - deny
+
+Response to SignTxRequest. When denying a request, there's no need to provide the transaction in return
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0x",
+    "to": null,
+    "gas": "0x0",
+    "gasPrice": "0x0",
+    "value": "0x0",
+    "nonce": "0x0",
+    "data": null
+  },
+  "approved": false
+}
+```
+### OnApproved - SignTransactionResult
+
+SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
+
+This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
+
+A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
+
+**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. 
+
+The `OnApproved` method cannot be responded to, it's purely informative
+
+Example:
+```json
+{
+  "raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+  "tx": {
+    "nonce": "0x64",
+    "gasPrice": "0x1",
+    "gas": "0x1",
+    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
+    "value": "0x1",
+    "input": "0x",
+    "v": "0x26",
+    "r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
+    "s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+    "hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
+  }
+}
+```
+### UserInputRequest
+
+Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
+
+Example:
+```json
+{
+  "prompt": "The question to ask the user",
+  "title": "The title here",
+  "isPassword": true
+}
+```
+### UserInputResponse
+
+Response to UserInputRequest
+
+Example:
+```json
+{
+  "text": "The textual response from user"
+}
+```
+### ListRequest
+
+Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es. 
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0xdeadbeef000000000000000000000000deadbeef",
+      "url": "keystore:///path/to/keyfile/a"
+    },
+    {
+      "address": "0x1111111122222222222233333333334444444444",
+      "url": "keystore:///path/to/keyfile/b"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### ListResponse
+
+Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "url": ".. ignored .."
+    },
+    {
+      "address": "0xffffffffffffffffffffffffffffffffffffffff",
+      "url": ""
+    }
+  ]
+}
+```

cmd/clef/extapi_changelog.md

@@ -1,6 +1,28 @@
 ### Changelog for external API
 
+### 6.0.0
 
+* `New` was changed to deliver only an address, not the full `Account` data
+* `Export` was moved from External API to the UI Server API
+
+#### 5.0.0
+
+* The external `account_EcRecover`-method was reimplemented.
+* The external method `account_sign(address, data)` was replaced with `account_signData(contentType, address, data)`.
+The addition of `contentType` makes it possible to use the method for different types of objects, such as:
+  * signing data with an intended validator (not yet implemented)
+  * signing clique headers,
+  * signing plain personal messages,
+* The external method `account_signTypedData` implements [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and makes it possible to sign typed data.
+  
+#### 4.0.0
+
+* The external `account_Ecrecover`-method was removed. 
+* The external `account_Import`-method was removed.
+
+#### 3.0.0
+
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
 
 #### 2.0.0
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,94 @@
 ### Changelog for internal API (ui-api)
 
+### 6.0.0 
+
+Removed `password` from responses to operations which require them. This is for two reasons, 
+
+- Consistency between how rulesets operate and how manual processing works. A rule can `Approve` but require the actual password to be stored in the clef storage. 
+With this change, the same stored password can be used even if rulesets are not enabled, but storage is. 
+- It also removes the usability-shortcut that a UI might otherwise want to implement; remembering passwords. Since we now will not require the 
+password on every `Approve`, there's no need for the UI to cache it locally. 
+  - In a future update, we'll likely add `clef_storePassword` to the internal API, so the user can store it via his UI (currently only CLI works). 
+
+Affected datatypes:
+- `SignTxResponse`
+- `SignDataResponse`
+- `NewAccountResponse`
+
+If `clef` requires a password, the `OnInputRequired` will be used to collect it. 
+
+
+### 5.0.0 
+
+Changed the namespace format to adhere to the legacy ethereum format: `name_methodName`. Changes:
+
+* `ApproveTx` -> `ui_approveTx`
+* `ApproveSignData` -> `ui_approveSignData`
+* `ApproveExport` -> `removed`
+* `ApproveImport`  -> `removed`
+* `ApproveListing`  -> `ui_approveListing`
+* `ApproveNewAccount`  -> `ui_approveNewAccount`
+* `ShowError` -> `ui_showError`
+* `ShowInfo` -> `ui_showInfo`
+* `OnApprovedTx` -> `ui_onApprovedTx`
+* `OnSignerStartup` -> `ui_onSignerStartup`
+* `OnInputRequired` -> `ui_onInputRequired`
+
+
+### 4.0.0
+
+* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
+  - `clef_listWallets` 
+  - `clef_listAccounts`
+  - `clef_listWallets`
+  - `clef_deriveAccount`
+  - `clef_importRawKey`
+  - `clef_openWallet`
+  - `clef_chainId`
+  - `clef_setChainId`
+  - `clef_export`
+  - `clef_import`
+ 
+* The type `Account` was modified (the json-field `type` was removed), to consist of 
+
+```golang
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+```
+
+
+### 3.2.0
+
+* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
+
+> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
+> 
+>  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
+### 3.1.0
+
+* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
+
+### 3.0.0
+
+* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
+
+### 2.1.0
+
+* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
+
+The following structures are used:
+```golang
+       UserInputRequest struct {
+               Prompt     string `json:"prompt"`
+               Title      string `json:"title"`
+               IsPassword bool   `json:"isPassword"`
+       }
+       UserInputResponse struct {
+               Text string `json:"text"`
+       }
+
 ### 2.0.0
 
 * Modify how `call_info` on a transaction is conveyed. New format:

cmd/clef/main.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/big"
 	"os"
 	"os/signal"
 	"os/user"
@@ -35,11 +36,19 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/console"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/signer/core"
 	"github.com/ethereum/go-ethereum/signer/rules"
@@ -47,12 +56,6 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-// ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "2.0.0"
-
-// InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "2.0.0"
-
 const legalWarning = `
 WARNING! 
 
@@ -70,6 +73,10 @@ var (
 		Value: 4,
 		Usage: "log level to emit to the screen",
 	}
+	advancedMode = cli.BoolFlag{
+		Name:  "advanced",
+		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
+	}
 	keystoreFlag = cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@@ -80,6 +87,11 @@ var (
 		Value: DefaultConfigDir(),
 		Usage: "Directory for Clef configuration",
 	}
+	chainIdFlag = cli.Int64Flag{
+		Name:  "chainid",
+		Value: params.MainnetChainConfig.ChainID.Int64(),
+		Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
+	}
 	rpcPortFlag = cli.IntFlag{
 		Name:  "rpcport",
 		Usage: "HTTP-RPC server listening port",
@@ -87,7 +99,7 @@ var (
 	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
-		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
 	dBFlag = cli.StringFlag{
 		Name:  "4bytedb",
@@ -107,7 +119,7 @@ var (
 	ruleFlag = cli.StringFlag{
 		Name:  "rules",
 		Usage: "Enable rule-engine",
-		Value: "rules.json",
+		Value: "",
 	}
 	stdiouiFlag = cli.BoolFlag{
 		Name: "stdio-ui",
@@ -151,21 +163,27 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
-		Name:      "addpw",
+	setCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(setCredential),
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
-`,
-	}
+`}
+	gendocCommand = cli.Command{
+		Action: GenDoc,
+		Name:   "gendoc",
+		Usage:  "Generate documentation about json-rpc format",
+		Description: `
+The gendoc generates example structures of the json-rpc communication types.
+`}
 )
 
 func init() {
@@ -175,7 +193,7 @@ func init() {
 		logLevelFlag,
 		keystoreFlag,
 		configdirFlag,
-		utils.NetworkIdFlag,
+		chainIdFlag,
 		utils.LightKDFFlag,
 		utils.NoUSBFlag,
 		utils.RPCListenAddrFlag,
@@ -191,9 +209,10 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
+		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
 
 }
 func main() {
@@ -207,25 +226,45 @@ func initializeSecrets(c *cli.Context) error {
 	if err := initialize(c); err != nil {
 		return err
 	}
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
 
 	masterSeed := make([]byte, 256)
-	n, err := io.ReadFull(rand.Reader, masterSeed)
+	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
 		return err
 	}
-	if n != len(masterSeed) {
+	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
+
+	n, p := keystore.StandardScryptN, keystore.StandardScryptP
+	if c.GlobalBool(utils.LightKDFFlag.Name) {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	var password string
+	for {
+		password = getPassPhrase(text, true)
+		if err := core.ValidatePasswordFormat(password); err != nil {
+			fmt.Printf("invalid password: %v\n", err)
+		} else {
+			break
+		}
+	}
+	cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt master seed: %v", err)
+	}
+
 	err = os.Mkdir(configDir, 0700)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "secrets.dat")
+	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0700)
+	err = ioutil.WriteFile(location, cipherSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -250,11 +289,11 @@ func attestFile(ctx *cli.Context) error {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -266,38 +305,36 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
-	value := ""
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
+	pwStorage.Put(address, password)
+	log.Info("Credential store updated", "key", address)
 	return nil
 }
 
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
 		fmt.Fprintf(logOutput, legalWarning)
@@ -316,33 +353,32 @@ func signer(c *cli.Context) error {
 		return err
 	}
 	var (
-		ui core.SignerUI
+		ui core.UIClientAPI
 	)
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
 		log.Info("Using CLI as UI-channel")
 		ui = core.NewCommandlineUI()
 	}
-	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	fourByteDb := c.GlobalString(dBFlag.Name)
+	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
 
 	var (
-		api core.ExternalAPI
+		api       core.ExternalAPI
+		pwStorage storage.Storage = &storage.NoStorage{}
 	)
 
-	configDir := c.String(configdirFlag.Name)
-	if stretchedKey, err := readMasterKey(c); err != nil {
-		log.Info("No master seed provided, rules disabled")
+	configDir := c.GlobalString(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c, ui); err != nil {
+		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
-
-		if err != nil {
-			utils.Fatalf(err.Error())
-		}
 		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 
 		// Generate domain specific keys
@@ -351,45 +387,52 @@ func signer(c *cli.Context) error {
 		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
 
 		// Initialize the encrypted storages
-		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
 		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
-		if err != nil {
-			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
-		} else {
-			hasher := sha256.New()
-			hasher.Write(ruleJS)
-			shasum := hasher.Sum(nil)
-			storedShasum := configStorage.Get("ruleset_sha256")
-			if storedShasum != hex.EncodeToString(shasum) {
-				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
+			ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFile))
+			if err != nil {
+				log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 			} else {
-				// Initialize rules
-				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
-				if err != nil {
-					utils.Fatalf(err.Error())
+				shasum := sha256.Sum256(ruleJS)
+				foundShaSum := hex.EncodeToString(shasum[:])
+				storedShasum := configStorage.Get("ruleset_sha256")
+				if storedShasum != foundShaSum {
+					log.Info("Could not validate ruleset hash, rules not enabled", "got", foundShaSum, "expected", storedShasum)
+				} else {
+					// Initialize rules
+					ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage)
+					if err != nil {
+						utils.Fatalf(err.Error())
+					}
+					ruleEngine.Init(string(ruleJS))
+					ui = ruleEngine
+					log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
 				}
-				ruleEngine.Init(string(ruleJS))
-				ui = ruleEngine
-				log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
 			}
 		}
 	}
+	var (
+		chainId  = c.GlobalInt64(chainIdFlag.Name)
+		ksLoc    = c.GlobalString(keystoreFlag.Name)
+		lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
+		advanced = c.GlobalBool(advancedMode.Name)
+		nousb    = c.GlobalBool(utils.NoUSBFlag.Name)
+	)
+	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
+		"light-kdf", lightKdf, "advanced", advanced)
+	am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
+	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced, pwStorage)
 
-	apiImpl := core.NewSignerAPI(
-		c.Int64(utils.NetworkIdFlag.Name),
-		c.String(keystoreFlag.Name),
-		c.Bool(utils.NoUSBFlag.Name),
-		ui, db,
-		c.Bool(utils.LightKDFFlag.Name))
-
+	// Establish the bidirectional communication, by creating a new UI backend and registering
+	// it with the UI.
+	ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
 	api = apiImpl
-
 	// Audit logging
-	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -408,13 +451,13 @@ func signer(c *cli.Context) error {
 			Service:   api,
 			Version:   "1.0"},
 	}
-	if c.Bool(utils.RPCEnabledFlag.Name) {
+	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
 
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
 		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
@@ -428,9 +471,9 @@ func signer(c *cli.Context) error {
 		}()
 
 	}
-	if !c.Bool(utils.IPCDisabledFlag.Name) {
+	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
-			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+			ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
 		} else {
 			ipcapiURL = filepath.Join(configDir, "clef.ipc")
 		}
@@ -447,14 +490,14 @@ func signer(c *cli.Context) error {
 
 	}
 
-	if c.Bool(testFlag.Name) {
+	if c.GlobalBool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
 	ui.OnSignerStartup(core.StartupInfo{
 		Info: map[string]interface{}{
-			"extapi_version": ExternalAPIVersion,
-			"intapi_version": InternalAPIVersion,
+			"extapi_version": core.ExternalAPIVersion,
+			"intapi_version": core.InternalAPIVersion,
 			"extapi_http":    extapiURL,
 			"extapi_ipc":     ipcapiURL,
 		},
@@ -488,7 +531,12 @@ func DefaultConfigDir() string {
 		if runtime.GOOS == "darwin" {
 			return filepath.Join(home, "Library", "Signer")
 		} else if runtime.GOOS == "windows" {
-			return filepath.Join(home, "AppData", "Roaming", "Signer")
+			appdata := os.Getenv("APPDATA")
+			if appdata != "" {
+				return filepath.Join(appdata, "Signer")
+			} else {
+				return filepath.Join(home, "AppData", "Roaming", "Signer")
+			}
 		} else {
 			return filepath.Join(home, ".clef")
 		}
@@ -506,48 +554,64 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.String(configdirFlag.Name)
+		configDir = ctx.GlobalString(configdirFlag.Name)
 	)
-	if ctx.IsSet(signerSecretFlag.Name) {
-		file = ctx.String(signerSecretFlag.Name)
+	if ctx.GlobalIsSet(signerSecretFlag.Name) {
+		file = ctx.GlobalString(signerSecretFlag.Name)
 	} else {
-		file = filepath.Join(configDir, "secrets.dat")
+		file = filepath.Join(configDir, "masterseed.json")
 	}
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	masterKey, err := ioutil.ReadFile(file)
+	cipherKey, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	if len(masterKey) < 256 {
-		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	var password string
+	// If ui is not nil, get the password from ui.
+	if ui != nil {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Master Password",
+			Prompt:     "Please enter the password to decrypt the master seed",
+			IsPassword: true})
+		if err != nil {
+			return nil, err
+		}
+		password = resp.Text
+	} else {
+		password = getPassPhrase("Decrypt master seed of clef", false)
 	}
+	masterSeed, err := decryptSeed(cipherKey, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt the master seed of clef")
+	}
+	if len(masterSeed) < 256 {
+		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
+	}
+
 	// Create vault location
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
 	if err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	//!TODO, use KDF to stretch the master key
-	//			stretched_key := stretch_key(master_key)
-
-	return masterKey, nil
+	return masterSeed, nil
 }
 
 // checkFile is a convenience function to check if a file
 // * exists
-// * is mode 0600
+// * is mode 0400
 func checkFile(filename string) error {
 	info, err := os.Stat(filename)
 	if err != nil {
 		return fmt.Errorf("failed stat on %s: %v", filename, err)
 	}
 	// Check the unix permission bits
-	if info.Mode().Perm()&077 != 0 {
+	if info.Mode().Perm()&0377 != 0 {
 		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
 	}
 	return nil
@@ -574,43 +638,325 @@ func testExternalUI(api *core.SignerAPI) {
 	ctx := context.WithValue(context.Background(), "remote", "clef binary")
 	ctx = context.WithValue(ctx, "scheme", "in-proc")
 	ctx = context.WithValue(ctx, "local", "main")
-
 	errs := make([]string, 0)
 
-	api.UI.ShowInfo("Testing 'ShowInfo'")
-	api.UI.ShowError("Testing 'ShowError'")
+	a := common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
 
-	checkErr := func(method string, err error) {
-		if err != nil && err != core.ErrRequestDenied {
-			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+	queryUser := func(q string) string {
+		resp, err := api.UI.OnInputRequired(core.UserInputRequest{
+			Title:  "Testing",
+			Prompt: q,
+		})
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+		return resp.Text
+	}
+	expectResponse := func(testcase, question, expect string) {
+		if got := queryUser(question); got != expect {
+			errs = append(errs, fmt.Sprintf("%s: got %v, expected %v", testcase, got, expect))
 		}
 	}
-	var err error
-
-	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
-	checkErr("SignTransaction", err)
-	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
-	checkErr("Sign", err)
-	_, err = api.List(ctx)
-	checkErr("List", err)
-	_, err = api.New(ctx)
-	checkErr("New", err)
-	_, err = api.Export(ctx, common.Address{})
-	checkErr("Export", err)
-	_, err = api.Import(ctx, json.RawMessage{})
-	checkErr("Import", err)
-
-	api.UI.ShowInfo("Tests completed")
-
-	if len(errs) > 0 {
-		log.Error("Got errors")
-		for _, e := range errs {
-			log.Error(e)
+	expectApprove := func(testcase string, err error) {
+		if err == nil || err == accounts.ErrUnknownAccount {
+			return
+		}
+		errs = append(errs, fmt.Sprintf("%v: expected no error, got %v", testcase, err.Error()))
+	}
+	expectDeny := func(testcase string, err error) {
+		if err == nil || err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: expected ErrRequestDenied, got %v", testcase, err))
 		}
-	} else {
-		log.Info("No errors")
 	}
 
+	// Test display of info and error
+	{
+		api.UI.ShowInfo("If you see this message, enter 'yes' to next question")
+		expectResponse("showinfo", "Did you see the message? [yes/no]", "yes")
+		api.UI.ShowError("If you see this message, enter 'yes' to the next question")
+		expectResponse("showerror", "Did you see the message? [yes/no]", "yes")
+	}
+	{ // Sign data test - clique header
+		api.UI.ShowInfo("Please approve the next request for signing a clique header")
+		cliqueHeader := types.Header{
+			common.HexToHash("0000H45H"),
+			common.HexToHash("0000H45H"),
+			common.HexToAddress("0000H45H"),
+			common.HexToHash("0000H00H"),
+			common.HexToHash("0000H45H"),
+			common.HexToHash("0000H45H"),
+			types.Bloom{},
+			big.NewInt(1337),
+			big.NewInt(1337),
+			1338,
+			1338,
+			big.NewInt(1338),
+			[]byte("Extra data Extra data Extra data  Extra data  Extra data  Extra data  Extra data Extra data"),
+			common.HexToHash("0x0000H45H"),
+			types.BlockNonce{},
+		}
+		cliqueRlp, err := rlp.EncodeToBytes(cliqueHeader)
+		if err != nil {
+			utils.Fatalf("Should not error: %v", err)
+		}
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		_, err = api.SignData(ctx, accounts.MimetypeClique, *addr, hexutil.Encode(cliqueRlp))
+		expectApprove("signdata - clique header", err)
+	}
+	{ // Sign data test - plain text
+		api.UI.ShowInfo("Please approve the next request for signing text")
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		_, err := api.SignData(ctx, accounts.MimetypeTextPlain, *addr, hexutil.Encode([]byte("hello world")))
+		expectApprove("signdata - text", err)
+	}
+	{ // Sign data test - plain text reject
+		api.UI.ShowInfo("Please deny the next request for signing text")
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		_, err := api.SignData(ctx, accounts.MimetypeTextPlain, *addr, hexutil.Encode([]byte("hello world")))
+		expectDeny("signdata - text", err)
+	}
+	{ // Sign transaction
+
+		api.UI.ShowInfo("Please reject next transaction")
+		data := hexutil.Bytes([]byte{})
+		to := common.NewMixedcaseAddress(a)
+		tx := core.SendTxArgs{
+			Data:     &data,
+			Nonce:    0x1,
+			Value:    hexutil.Big(*big.NewInt(6)),
+			From:     common.NewMixedcaseAddress(a),
+			To:       &to,
+			GasPrice: hexutil.Big(*big.NewInt(5)),
+			Gas:      1000,
+			Input:    nil,
+		}
+		_, err := api.SignTransaction(ctx, tx, nil)
+		expectDeny("signtransaction [1]", err)
+		expectResponse("signtransaction [2]", "Did you see any warnings for the last transaction? (yes/no)", "no")
+	}
+	{ // Listing
+		api.UI.ShowInfo("Please reject listing-request")
+		_, err := api.List(ctx)
+		expectDeny("list", err)
+	}
+	{ // Import
+		api.UI.ShowInfo("Please reject new account-request")
+		_, err := api.New(ctx)
+		expectDeny("newaccount", err)
+	}
+	{ // Metadata
+		api.UI.ShowInfo("Please check if you see the Origin in next listing (approve or deny)")
+		api.List(context.WithValue(ctx, "Origin", "origin.com"))
+		expectResponse("metadata - origin", "Did you see origin (origin.com)? [yes/no] ", "yes")
+	}
+
+	for _, e := range errs {
+		log.Error(e)
+	}
+	result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n"))
+	api.UI.ShowInfo(result)
+
+}
+
+// getPassPhrase retrieves the password associated with clef, either fetched
+// from a list of preloaded passphrases, or requested interactively from the user.
+// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
+func getPassPhrase(prompt string, confirmation bool) string {
+	fmt.Println(prompt)
+	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	if err != nil {
+		utils.Fatalf("Failed to read passphrase: %v", err)
+	}
+	if confirmation {
+		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+		}
+		if password != confirm {
+			utils.Fatalf("Passphrases do not match")
+		}
+	}
+	return password
+}
+
+type encryptedSeedStorage struct {
+	Description string              `json:"description"`
+	Version     int                 `json:"version"`
+	Params      keystore.CryptoJSON `json:"params"`
+}
+
+// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
+// to encrypt the master seed
+func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
+	cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
+}
+
+// decryptSeed decrypts the master seed
+func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
+	var encSeed encryptedSeedStorage
+	if err := json.Unmarshal(keyjson, &encSeed); err != nil {
+		return nil, err
+	}
+	if encSeed.Version != 1 {
+		log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
+	}
+	seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
+	if err != nil {
+		return nil, err
+	}
+	return seed, err
+}
+
+// GenDoc outputs examples of all structures used in json-rpc communication
+func GenDoc(ctx *cli.Context) {
+
+	var (
+		a    = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
+		b    = common.HexToAddress("0x1111111122222222222233333333334444444444")
+		meta = core.Metadata{
+			Scheme:    "http",
+			Local:     "localhost:8545",
+			Origin:    "www.malicious.ru",
+			Remote:    "localhost:9999",
+			UserAgent: "Firefox 3.2",
+		}
+		output []string
+		add    = func(name, desc string, v interface{}) {
+			if data, err := json.MarshalIndent(v, "", "  "); err == nil {
+				output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data))
+			} else {
+				log.Error("Error generating output", err)
+			}
+		}
+	)
+
+	{ // Sign plain text request
+		desc := "SignDataRequest contains information about a pending request to sign some data. " +
+			"The data to be signed can be of various types, defined by content-type. Clef has done most " +
+			"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
+			"the user with the contents of the `message`"
+		sighash, msg := accounts.TextAndHash([]byte("hello world"))
+		message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
+
+		add("SignDataRequest", desc, &core.SignDataRequest{
+			Address:     common.NewMixedcaseAddress(a),
+			Meta:        meta,
+			ContentType: accounts.MimetypeTextPlain,
+			Rawdata:     []byte(msg),
+			Message:     message,
+			Hash:        sighash})
+	}
+	{ // Sign plain text response
+		add("SignDataResponse - approve", "Response to SignDataRequest",
+			&core.SignDataResponse{Approved: true})
+		add("SignDataResponse - deny", "Response to SignDataRequest",
+			&core.SignDataResponse{})
+	}
+	{ // Sign transaction request
+		desc := "SignTxRequest contains information about a pending request to sign a transaction. " +
+			"Aside from the transaction itself, there is also a `call_info`-struct. That struct contains " +
+			"messages of various types, that the user should be informed of." +
+			"\n\n" +
+			"As in any request, it's important to consider that the `meta` info also contains untrusted data." +
+			"\n\n" +
+			"The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, " +
+			"they must be identical, otherwise an error is generated. " +
+			"However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)"
+
+		data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04})
+		add("SignTxRequest", desc, &core.SignTxRequest{
+			Meta: meta,
+			Callinfo: []core.ValidationInfo{
+				{"Warning", "Something looks odd, show this message as a warning"},
+				{"Info", "User should see this aswell"},
+			},
+			Transaction: core.SendTxArgs{
+				Data:     &data,
+				Nonce:    0x1,
+				Value:    hexutil.Big(*big.NewInt(6)),
+				From:     common.NewMixedcaseAddress(a),
+				To:       nil,
+				GasPrice: hexutil.Big(*big.NewInt(5)),
+				Gas:      1000,
+				Input:    nil,
+			}})
+	}
+	{ // Sign tx response
+		data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
+		add("SignTxResponse - approve", "Response to request to sign a transaction. This response needs to contain the `transaction`"+
+			", because the UI is free to make modifications to the transaction.",
+			&core.SignTxResponse{Approved: true,
+				Transaction: core.SendTxArgs{
+					Data:     &data,
+					Nonce:    0x4,
+					Value:    hexutil.Big(*big.NewInt(6)),
+					From:     common.NewMixedcaseAddress(a),
+					To:       nil,
+					GasPrice: hexutil.Big(*big.NewInt(5)),
+					Gas:      1000,
+					Input:    nil,
+				}})
+		add("SignTxResponse - deny", "Response to SignTxRequest. When denying a request, there's no need to "+
+			"provide the transaction in return",
+			&core.SignTxResponse{})
+	}
+	{ // WHen a signed tx is ready to go out
+		desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
+			"\n\n" +
+			"This occurs _after_ successful completion of the entire signing procedure, but right before the signed " +
+			"transaction is passed to the external caller. This method (and data) can be used by the UI to signal " +
+			"to the user that the transaction was signed, but it is primarily useful for ruleset implementations." +
+			"\n\n" +
+			"A ruleset that implements a rate limitation needs to know what transactions are sent out to the external " +
+			"interface. By hooking into this methods, the ruleset can maintain track of that count." +
+			"\n\n" +
+			"**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time" +
+			" (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. " +
+			"\n\n" +
+			"The `OnApproved` method cannot be responded to, it's purely informative"
+
+		rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
+		var tx types.Transaction
+		rlp.DecodeBytes(rlpdata, &tx)
+		add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
+
+	}
+	{ // User input
+		add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
+			&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
+		add("UserInputResponse", "Response to UserInputRequest",
+			&core.UserInputResponse{Text: "The textual response from user"})
+	}
+	{ // List request
+		add("ListRequest", "Sent when a request has been made to list addresses. The UI is provided with the "+
+			"full `account`s, including local directory names. Note: this information is not passed back to the external caller, "+
+			"who only sees the `address`es. ",
+			&core.ListRequest{
+				Meta: meta,
+				Accounts: []accounts.Account{
+					{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
+					{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
+			})
+
+		add("ListResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
+			"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
+			&core.ListResponse{
+				Accounts: []accounts.Account{
+					{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
+					{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
+				}})
+	}
+
+	fmt.Println(`## UI Client interface
+
+These data types are defined in the channel between clef and the UI`)
+	for _, elem := range output {
+		fmt.Println(elem)
+	}
 }
 
 /**

cmd/clef/tests/testsigner.js

@@ -0,0 +1,73 @@
+// This file is a test-utility for testing clef-functionality
+//
+// Start clef with
+//
+// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
+//
+// Start geth with
+//
+// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
+//
+// and in the console simply invoke
+//
+// > test()
+//
+// You can reload the file via `reload()`
+
+function reload(){
+	loadScript("./cmd/clef/tests/testsigner.js");
+}
+
+function init(){
+    if (typeof accts == 'undefined' || accts.length == 0){
+        accts = eth.accounts
+        console.log("Got accounts ", accts);
+    }
+}
+init()
+function testTx(){
+    if( accts && accts.length > 0) {
+        var a = accts[0]
+        var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
+        var v = parseInt(txdata.tx.v)
+        console.log("V value: ", v)
+        if (v == 37 || v == 38){
+            console.log("Mainnet 155-protected chainid was used")
+        }
+        if (v == 27 || v == 28){
+            throw new Error("Mainnet chainid was used, but without replay protection!")
+        }
+    }
+}
+function testSignText(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
+        console.log("signing response",  r)
+    }
+}
+function testClique(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
+        console.log("signing response",  r)
+        if( a != r){
+            throw new Error("Requested signing by "+a+ " but got sealer "+r)
+        }
+    }
+}
+
+function test(){
+    var tests = [
+        testTx,
+        testSignText,
+        testClique,
+    ]
+    for( i in tests){
+        try{
+            tests[i]()
+        }catch(err){
+            console.log(err)
+        }
+    }
+ }

cmd/clef/tutorial.md

@@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
 
 ## Creating rules
 
-Now, you can create a rule-file.
+Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
 
 ```javascript
 function ApproveListing(){
     return "Approve"
 }
 ```
-Get the `sha256` hash....
+
+Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
 ```text
 #sha256sum rules.js
 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
 ```
-...And then `attest` the file:
+...now `attest` the file...
 ```text
 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
 INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 ```
-At this point, we then start the signer with the rule-file:
 
+...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
 ```text
-#./signer --rules rules.json
+#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
+```
 
-INFO [02-21|12:15:18] Using CLI as UI-channel
-INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
-DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
-DEBUG[02-21|12:15:18] Ledger support enabled
-DEBUG[02-21|12:15:18] Trezor support enabled
-INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
-INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+At this point, we can start the signer with the rule-file:
+```text
+#./signer --rules rules.js --rpc
+
+INFO [09-25|20:28:11.866] Using CLI as UI-channel 
+INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
+DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[09-25|20:28:11.888] Trezor support enabled 
+INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
+INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-* extapi_version : 2.0.0
-* intapi_version : 1.2.0
-
 ```
 
 Any list-requests will now be auto-approved by our rule-file.
@@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
 
 ## Adding credentials
 
-In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
 
 INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
 ```
 ## More advanced rules
 
-Now let's update the rules to make use of credentials
+Now let's update the rules to make use of credentials:
 
 ```javascript
 function ApproveListing(){
@@ -134,13 +142,15 @@ function ApproveSignData(r){
 }
 
 ```
-In this example,
-* any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`,
-    * and auto-rejected if it does not.
-    * Any other signing-requests will be passed along for manual approve/reject.
+In this example:
+* Any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`
+    * auto-rejected if it does not.
+* Any other signing-requests will be passed along for manual approve/reject.
 
-..attest the new file
+_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+
+Attest the new file...
 ```text
 #sha256sum rules.js
 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
@@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661da
 And start the signer:
 
 ```
-#./signer --rules rules.js
+#./signer --rules rules.js --rpc
 
-INFO [02-21|14:41:56] Using CLI as UI-channel
-INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
-DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
-DEBUG[02-21|14:41:56] Ledger support enabled
-DEBUG[02-21|14:41:56] Trezor support enabled
-INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
-INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+INFO [09-25|21:02:16.450] Using CLI as UI-channel 
+INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
+DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
+DEBUG[09-25|21:02:16.473] Ledger support enabled 
+DEBUG[09-25|21:02:16.475] Trezor support enabled 
+INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
+DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
+INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
 * extapi_version : 2.0.0
-* intapi_version : 1.2.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
 ```
+
 And then test signing, once with `bazonk` and once without:
 
 ```
@@ -190,9 +203,9 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail audit.log -n 4
+#tail -n 4 audit.log
 t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
-```
+```

cmd/ethkey/README.md

@@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
 make sure to use this feature with great caution!
 
 
-### `ethkey sign <keyfile> <message/file>`
+### `ethkey signmessage <keyfile> <message/file>`
 
 Sign the message with a keyfile.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the `--msgfile` flag.
 
 
-### `ethkey verify <address> <signature> <message/file>`
+### `ethkey verifymessage <address> <signature> <message/file>`
 
 Verify the signature of the message.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the --msgfile flag.
+
+
+### `ethkey changepassphrase <keyfile>`
+
+Change the passphrase of a keyfile.
+use the `--newpasswordfile` to point to the new password file.
 
 
 ## Passphrases
 
 For every command that uses a keyfile, you will be prompted to provide the 
 passphrase for decrypting the keyfile.  To avoid this message, it is possible
-to pass the passphrase by using the `--passphrase` flag pointing to a file that
+to pass the passphrase by using the `--passwordfile` flag pointing to a file that
 contains the passphrase.
+
+## JSON
+
+In case you need to output the result in a JSON format, you shall by using the `--json` flag.

cmd/evm/disasm.go

@@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
 		return err
 	}
 
-	code := strings.TrimSpace(string(in[:]))
+	code := strings.TrimSpace(string(in))
 	fmt.Printf("%v\n", code)
 	return asm.PrintDisassembled(code)
 }

cmd/evm/internal/compiler/compiler.go

@@ -25,7 +25,7 @@ import (
 
 func Compile(fn string, src []byte, debug bool) (string, error) {
 	compiler := asm.NewCompiler(debug)
-	compiler.Feed(asm.Lex(fn, src, debug))
+	compiler.Feed(asm.Lex(src, debug))
 
 	bin, compileErrors := compiler.Compile()
 	if len(compileErrors) > 0 {

cmd/evm/main.go

@@ -110,6 +110,11 @@ var (
 		Name:  "nostack",
 		Usage: "disable stack output",
 	}
+	EVMInterpreterFlag = cli.StringFlag{
+		Name:  "vm.evm",
+		Usage: "External EVM configuration (default = built-in interpreter)",
+		Value: "",
+	}
 )
 
 func init() {
@@ -133,6 +138,7 @@ func init() {
 		ReceiverFlag,
 		DisableMemoryFlag,
 		DisableStackFlag,
+		EVMInterpreterFlag,
 	}
 	app.Commands = []cli.Command{
 		compileCommand,

cmd/evm/runner.go

@@ -31,10 +31,10 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	cli "gopkg.in/urfave/cli.v1"
@@ -80,16 +80,16 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer      vm.Tracer
-		debugLogger *vm.StructLogger
-		statedb     *state.StateDB
-		chainConfig *params.ChainConfig
-		sender      = common.BytesToAddress([]byte("sender"))
-		receiver    = common.BytesToAddress([]byte("receiver"))
-		blockNumber uint64
+		tracer        vm.Tracer
+		debugLogger   *vm.StructLogger
+		statedb       *state.StateDB
+		chainConfig   *params.ChainConfig
+		sender        = common.BytesToAddress([]byte("sender"))
+		receiver      = common.BytesToAddress([]byte("receiver"))
+		genesisConfig *core.Genesis
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
-		tracer = NewJSONLogger(logconfig, os.Stdout)
+		tracer = vm.NewJSONLogger(logconfig, os.Stdout)
 	} else if ctx.GlobalBool(DebugFlag.Name) {
 		debugLogger = vm.NewStructLogger(logconfig)
 		tracer = debugLogger
@@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
-		db := ethdb.NewMemDatabase()
+		genesisConfig = gen
+		db := rawdb.NewMemoryDatabase()
 		genesis := gen.ToBlock(db)
 		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
-		blockNumber = gen.Number
 	} else {
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
+		genesisConfig = new(core.Genesis)
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@@ -156,16 +157,23 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	initialGas := ctx.GlobalUint64(GasFlag.Name)
+	if genesisConfig.GasLimit != 0 {
+		initialGas = genesisConfig.GasLimit
+	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
 		GasLimit:    initialGas,
 		GasPrice:    utils.GlobalBig(ctx, PriceFlag.Name),
 		Value:       utils.GlobalBig(ctx, ValueFlag.Name),
-		BlockNumber: new(big.Int).SetUint64(blockNumber),
+		Difficulty:  genesisConfig.Difficulty,
+		Time:        new(big.Int).SetUint64(genesisConfig.Timestamp),
+		Coinbase:    genesisConfig.Coinbase,
+		BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
 		EVMConfig: vm.Config{
-			Tracer: tracer,
-			Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
+			Tracer:         tracer,
+			Debug:          ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
+			EVMInterpreter: ctx.GlobalString(EVMInterpreterFlag.Name),
 		},
 	}
 
@@ -199,6 +207,7 @@ func runCmd(ctx *cli.Context) error {
 	execTime := time.Since(tstart)
 
 	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
 		fmt.Println(string(statedb.Dump()))
 	}

cmd/evm/staterunner.go

@@ -68,7 +68,7 @@ func stateTestCmd(ctx *cli.Context) error {
 	)
 	switch {
 	case ctx.GlobalBool(MachineFlag.Name):
-		tracer = NewJSONLogger(config, os.Stderr)
+		tracer = vm.NewJSONLogger(config, os.Stderr)
 
 	case ctx.GlobalBool(DebugFlag.Name):
 		debugger = vm.NewStructLogger(config)
@@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
 			state, err := test.Run(st, cfg)
+			// print state root for evmlab tracing
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
 			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
@@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
-			// print state root for evmlab tracing (already committed above, so no need to delete objects again
-			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
-			}
 
 			results = append(results, *result)
 

cmd/faucet/faucet.go

@@ -54,8 +54,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
 	"golang.org/x/net/websocket"
@@ -157,7 +157,8 @@ func main() {
 	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
-	pass := string(blob)
+	// Delete trailing newline in password
+	pass := strings.TrimSuffix(string(blob), "\n")
 
 	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
 	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@@ -198,6 +199,8 @@ type faucet struct {
 
 	keystore *keystore.KeyStore // Keystore containing the single signer
 	account  accounts.Account   // Account funding user faucet requests
+	head     *types.Header      // Current head header of the faucet
+	balance  *big.Int           // Current balance of the faucet
 	nonce    uint64             // Current pending nonce of the faucet
 	price    *big.Int           // Current gas price to issue funds with
 
@@ -252,8 +255,10 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 		return nil, err
 	}
 	for _, boot := range enodes {
-		old, _ := discover.ParseNode(boot.String())
-		stack.Server().AddPeer(old)
+		old, err := enode.ParseV4(boot.String())
+		if err == nil {
+			stack.Server().AddPeer(old)
+		}
 	}
 	// Attach to the client and retrieve and interesting metadatas
 	api, err := stack.Attach()
@@ -277,7 +282,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 
 // close terminates the Ethereum connection and tears down the faucet.
 func (f *faucet) close() error {
-	return f.stack.Stop()
+	return f.stack.Close()
 }
 
 // listenAndServe registers the HTTP handlers for the faucet and boots it up
@@ -323,33 +328,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		nonce   uint64
 		err     error
 	)
-	for {
-		// Attempt to retrieve the stats, may error on no faucet connectivity
-		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-		head, err = f.client.HeaderByNumber(ctx, nil)
-		if err == nil {
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-			}
+	for head == nil || balance == nil {
+		// Retrieve the current stats cached by the faucet
+		f.lock.RLock()
+		if f.head != nil {
+			head = types.CopyHeader(f.head)
 		}
-		cancel()
+		if f.balance != nil {
+			balance = new(big.Int).Set(f.balance)
+		}
+		nonce = f.nonce
+		f.lock.RUnlock()
 
-		// If stats retrieval failed, wait a bit and retry
-		if err != nil {
-			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+		if head == nil || balance == nil {
+			// Report the faucet offline until initial stats are ready
+			if err = sendError(conn, errors.New("Faucet offline")); err != nil {
 				log.Warn("Failed to send faucet error to client", "err", err)
 				return
 			}
 			time.Sleep(3 * time.Second)
-			continue
 		}
-		// Initial stats reported successfully, proceed with user interaction
-		break
 	}
 	// Send over the initial stats and the latest header
 	if err = send(conn, map[string]interface{}{
-		"funds":    balance.Div(balance, ether),
+		"funds":    new(big.Int).Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
@@ -519,6 +521,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 	}
 }
 
+// refresh attempts to retrieve the latest header from the chain and extract the
+// associated faucet balance and nonce for connectivity caching.
+func (f *faucet) refresh(head *types.Header) error {
+	// Ensure a state update does not run for too long
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// If no header was specified, use the current chain head
+	var err error
+	if head == nil {
+		if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
+			return err
+		}
+	}
+	// Retrieve the balance, nonce and gas price from the current head
+	var (
+		balance *big.Int
+		nonce   uint64
+		price   *big.Int
+	)
+	if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if price, err = f.client.SuggestGasPrice(ctx); err != nil {
+		return err
+	}
+	// Everything succeeded, update the cached stats and eject old requests
+	f.lock.Lock()
+	f.head, f.balance = head, balance
+	f.price, f.nonce = price, nonce
+	for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+		f.reqs = f.reqs[1:]
+	}
+	f.lock.Unlock()
+
+	return nil
+}
+
 // loop keeps waiting for interesting events and pushes them out to connected
 // websockets.
 func (f *faucet) loop() {
@@ -536,45 +579,27 @@ func (f *faucet) loop() {
 	go func() {
 		for head := range update {
 			// New chain head arrived, query the current stats and stream to clients
-			var (
-				balance *big.Int
-				nonce   uint64
-				price   *big.Int
-				err     error
-			)
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-				if err == nil {
-					price, err = f.client.SuggestGasPrice(ctx)
-				}
+			timestamp := time.Unix(head.Time.Int64(), 0)
+			if time.Since(timestamp) > time.Hour {
+				log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
+				continue
 			}
-			cancel()
-
-			// If querying the data failed, try for the next block
-			if err != nil {
+			if err := f.refresh(head); err != nil {
 				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
 				continue
-			} else {
-				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
 			}
 			// Faucet state retrieved, update locally and send to clients
-			balance = new(big.Int).Div(balance, ether)
-
-			f.lock.Lock()
-			f.price, f.nonce = price, nonce
-			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
-				f.reqs = f.reqs[1:]
-			}
-			f.lock.Unlock()
-
 			f.lock.RLock()
+			log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
+
+			balance := new(big.Int).Div(f.balance, ether)
+			peers := f.stack.Server().PeerCount()
+
 			for _, conn := range f.conns {
 				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
-					"peers":    f.stack.Server().PeerCount(),
+					"peers":    peers,
 					"requests": f.reqs,
 				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)

cmd/geth/chaincmd.go

@@ -29,14 +29,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -190,8 +188,10 @@ func initGenesis(ctx *cli.Context) error {
 	}
 	// Open an initialise both full and light databases
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		chaindb, err := stack.OpenDatabase(name, 0, 0)
+		chaindb, err := stack.OpenDatabase(name, 0, 0, "")
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}
@@ -199,6 +199,7 @@ func initGenesis(ctx *cli.Context) error {
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
+		chaindb.Close()
 		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
 	return nil
@@ -209,8 +210,10 @@ func importChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
-	defer chainDb.Close()
+	defer stack.Close()
+
+	chain, db := utils.MakeChain(ctx, stack)
+	defer db.Close()
 
 	// Start periodically gathering memory profiles
 	var peakMemAlloc, peakMemSys uint64
@@ -245,23 +248,18 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing
-	db := chainDb.(*ethdb.LDBDatabase)
-
-	stats, err := db.LDB().GetProperty("leveldb.stats")
+	stats, err := db.Stat("leveldb.stats")
 	if err != nil {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
 
-	ioStats, err := db.LDB().GetProperty("leveldb.iostats")
+	ioStats, err := db.Stat("leveldb.iostats")
 	if err != nil {
 		utils.Fatalf("Failed to read database iostats: %v", err)
 	}
 	fmt.Println(ioStats)
 
-	fmt.Printf("Trie cache misses:  %d\n", trie.CacheMisses())
-	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
-
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)
 	runtime.ReadMemStats(mem)
@@ -278,23 +276,22 @@ func importChain(ctx *cli.Context) error {
 	// Compact the entire database to more accurately measure disk io and print the stats
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
-	if err = db.LDB().CompactRange(util.Range{}); err != nil {
+	if err = db.Compact(nil, nil); err != nil {
 		utils.Fatalf("Compaction failed: %v", err)
 	}
 	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
 
-	stats, err = db.LDB().GetProperty("leveldb.stats")
+	stats, err = db.Stat("leveldb.stats")
 	if err != nil {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
 
-	ioStats, err = db.LDB().GetProperty("leveldb.iostats")
+	ioStats, err = db.Stat("leveldb.iostats")
 	if err != nil {
 		utils.Fatalf("Failed to read database iostats: %v", err)
 	}
 	fmt.Println(ioStats)
-
 	return nil
 }
 
@@ -303,6 +300,8 @@ func exportChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	chain, _ := utils.MakeChain(ctx, stack)
 	start := time.Now()
 
@@ -336,13 +335,15 @@ func importPreimages(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	defer stack.Close()
 
+	db := utils.MakeChainDatabase(ctx, stack)
 	start := time.Now()
-	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
-		utils.Fatalf("Export error: %v\n", err)
+
+	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
+		utils.Fatalf("Import error: %v\n", err)
 	}
-	fmt.Printf("Export done in %v\n", time.Since(start))
+	fmt.Printf("Import done in %v\n", time.Since(start))
 	return nil
 }
 
@@ -352,10 +353,12 @@ func exportPreimages(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	defer stack.Close()
 
+	db := utils.MakeChainDatabase(ctx, stack)
 	start := time.Now()
-	if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
+
+	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
 	fmt.Printf("Export done in %v\n", time.Since(start))
@@ -369,13 +372,14 @@ func copyDb(ctx *cli.Context) error {
 	}
 	// Initialize a new chain for the running node to sync into
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	defer stack.Close()
 
+	chain, chainDb := utils.MakeChain(ctx, stack)
 	syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
 	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
 
 	// Create a source peer to satisfy downloader requests from
-	db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256)
+	db, err := rawdb.NewLevelDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256, "")
 	if err != nil {
 		return err
 	}
@@ -402,11 +406,10 @@ func copyDb(ctx *cli.Context) error {
 	// Compact the entire database to remove any sync overhead
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
-	if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil {
+	if err = db.Compact(nil, nil); err != nil {
 		utils.Fatalf("Compaction failed: %v", err)
 	}
 	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
-
 	return nil
 }
 
@@ -441,6 +444,8 @@ func removeDB(ctx *cli.Context) error {
 
 func dump(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	chain, chainDb := utils.MakeChain(ctx, stack)
 	for _, arg := range ctx.Args() {
 		var block *types.Block

cmd/geth/config.go

@@ -20,7 +20,7 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	"io"
+	"math/big"
 	"os"
 	"reflect"
 	"unicode"
@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/dashboard"
 	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/graphql"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
@@ -124,6 +125,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	}
 
 	// Apply flags.
+	utils.SetULC(ctx, &cfg.Eth)
 	utils.SetNodeConfig(ctx, &cfg.Node)
 	stack, err := node.New(&cfg.Node)
 	if err != nil {
@@ -152,7 +154,9 @@ func enableWhisper(ctx *cli.Context) bool {
 
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
-
+	if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) {
+		cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name))
+	}
 	utils.RegisterEthService(stack, &cfg.Eth)
 
 	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
@@ -168,9 +172,19 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
 			cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
 		}
+		if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
+			cfg.Shh.RestrictConnectionBetweenLightClients = true
+		}
 		utils.RegisterShhService(stack, &cfg.Shh)
 	}
 
+	// Configure GraphQL if required
+	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
+		if err := graphql.RegisterGraphQLService(stack, cfg.Node.GraphQLEndpoint(), cfg.Node.GraphQLCors, cfg.Node.GraphQLVirtualHosts, cfg.Node.HTTPTimeouts); err != nil {
+			utils.Fatalf("Failed to register the Ethereum service: %v", err)
+		}
+	}
+
 	// Add the Ethereum Stats daemon if requested.
 	if cfg.Ethstats.URL != "" {
 		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
@@ -192,7 +206,17 @@ func dumpConfig(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	io.WriteString(os.Stdout, comment)
-	os.Stdout.Write(out)
+
+	dump := os.Stdout
+	if ctx.NArg() > 0 {
+		dump, err = os.OpenFile(ctx.Args().Get(0), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			return err
+		}
+		defer dump.Close()
+	}
+	dump.WriteString(comment)
+	dump.Write(out)
+
 	return nil
 }

cmd/geth/consolecmd.go

@@ -79,7 +79,7 @@ func localConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
-	defer node.Stop()
+	defer node.Close()
 
 	// Attach to the newly started node and start the JavaScript console
 	client, err := node.Attach()
@@ -180,7 +180,7 @@ func ephemeralConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
-	defer node.Stop()
+	defer node.Close()
 
 	// Attach to the newly started node and start the JavaScript console
 	client, err := node.Attach()

cmd/geth/dao_test.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -121,7 +120,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "geth", "chaindata")
-	db, err := ethdb.NewLDBDatabase(path, 0, 0)
+	db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
 	}

cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -32,14 +31,16 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 const (
@@ -61,6 +62,7 @@ var (
 		utils.BootnodesV5Flag,
 		utils.DataDirFlag,
 		utils.KeyStoreDirFlag,
+		utils.ExternalSignerFlag,
 		utils.NoUSBFlag,
 		utils.DashboardEnabledFlag,
 		utils.DashboardAddrFlag,
@@ -83,15 +85,23 @@ var (
 		utils.TxPoolAccountQueueFlag,
 		utils.TxPoolGlobalQueueFlag,
 		utils.TxPoolLifetimeFlag,
+		utils.ULCModeConfigFlag,
+		utils.OnlyAnnounceModeFlag,
+		utils.ULCTrustedNodesFlag,
+		utils.ULCMinTrustedFractionFlag,
 		utils.SyncModeFlag,
+		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.LightServFlag,
+		utils.LightBandwidthInFlag,
+		utils.LightBandwidthOutFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
+		utils.WhitelistFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
-		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
@@ -120,8 +130,10 @@ var (
 		utils.DeveloperPeriodFlag,
 		utils.TestnetFlag,
 		utils.RinkebyFlag,
+		utils.GoerliFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
+		utils.ConstantinopleOverrideFlag,
 		utils.RPCCORSDomainFlag,
 		utils.RPCVirtualHostsFlag,
 		utils.EthStatsURLFlag,
@@ -130,6 +142,8 @@ var (
 		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
+		utils.EWASMInterpreterFlag,
+		utils.EVMInterpreterFlag,
 		configFileFlag,
 	}
 
@@ -137,6 +151,11 @@ var (
 		utils.RPCEnabledFlag,
 		utils.RPCListenAddrFlag,
 		utils.RPCPortFlag,
+		utils.GraphQLEnabledFlag,
+		utils.GraphQLListenAddrFlag,
+		utils.GraphQLPortFlag,
+		utils.GraphQLCORSDomainFlag,
+		utils.GraphQLVirtualHostsFlag,
 		utils.RPCApiFlag,
 		utils.WSEnabledFlag,
 		utils.WSListenAddrFlag,
@@ -151,6 +170,7 @@ var (
 		utils.WhisperEnabledFlag,
 		utils.WhisperMaxMessageSizeFlag,
 		utils.WhisperMinPOWFlag,
+		utils.WhisperRestrictConnectionBetweenLightClientsFlag,
 	}
 
 	metricsFlags = []cli.Flag{
@@ -159,7 +179,7 @@ var (
 		utils.MetricsInfluxDBDatabaseFlag,
 		utils.MetricsInfluxDBUsernameFlag,
 		utils.MetricsInfluxDBPasswordFlag,
-		utils.MetricsInfluxDBHostTagFlag,
+		utils.MetricsInfluxDBTagsFlag,
 	}
 )
 
@@ -167,7 +187,7 @@ func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2018 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2019 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
@@ -206,8 +226,6 @@ func init() {
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-
 		logdir := ""
 		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
 			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")
@@ -262,6 +280,7 @@ func geth(ctx *cli.Context) error {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
 	node := makeFullNode(ctx)
+	defer node.Close()
 	startNode(ctx, node)
 	node.Wait()
 	return nil
@@ -277,13 +296,14 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	utils.StartNode(stack)
 
 	// Unlock any account specifically requested
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
-
-	passwords := utils.MakePasswordList(ctx)
-	unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
-	for i, account := range unlocks {
-		if trimmed := strings.TrimSpace(account); trimmed != "" {
-			unlockAccount(ctx, ks, trimmed, i, passwords)
+	if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
+		ks := keystores[0].(*keystore.KeyStore)
+		passwords := utils.MakePasswordList(ctx)
+		unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
+		for i, account := range unlocks {
+			if trimmed := strings.TrimSpace(account); trimmed != "" {
+				unlockAccount(ctx, ks, trimmed, i, passwords)
+			}
 		}
 	}
 	// Register wallet event handlers to open and auto-derive wallets
@@ -327,6 +347,32 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 			}
 		}
 	}()
+
+	// Spawn a standalone goroutine for status synchronization monitoring,
+	// close the node when synchronization is complete if user required.
+	if ctx.GlobalBool(utils.ExitWhenSyncedFlag.Name) {
+		go func() {
+			sub := stack.EventMux().Subscribe(downloader.DoneEvent{})
+			defer sub.Unsubscribe()
+			for {
+				event := <-sub.Chan()
+				if event == nil {
+					continue
+				}
+				done, ok := event.Data.(downloader.DoneEvent)
+				if !ok {
+					continue
+				}
+				if timestamp := time.Unix(done.Latest.Time.Int64(), 0); time.Since(timestamp) < 10*time.Minute {
+					log.Info("Synchronisation completed", "latestnum", done.Latest.Number, "latesthash", done.Latest.Hash(),
+						"age", common.PrettyAge(timestamp))
+					stack.Stop()
+				}
+
+			}
+		}()
+	}
+
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
 		// Mining only makes sense if a full Ethereum node is running

cmd/geth/monitorcmd.go

@@ -169,7 +169,7 @@ func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
 // resolveMetrics takes a list of input metric patterns, and resolves each to one
 // or more canonical metric names.
 func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
-	res := []string{}
+	var res []string
 	for _, pattern := range patterns {
 		res = append(res, resolveMetric(metrics, pattern, "")...)
 	}
@@ -179,7 +179,7 @@ func resolveMetrics(metrics map[string]interface{}, patterns []string) []string
 // resolveMetrics takes a single of input metric pattern, and resolves it to one
 // or more canonical metric names.
 func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
-	results := []string{}
+	var results []string
 
 	// If a nested metric was requested, recurse optionally branching (via comma)
 	parts := strings.SplitN(pattern, "/", 2)
@@ -215,7 +215,7 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
 // expandMetrics expands the entire tree of metrics into a flat list of paths.
 func expandMetrics(metrics map[string]interface{}, path string) []string {
 	// Iterate over all fields and expand individually
-	list := []string{}
+	var list []string
 	for name, metric := range metrics {
 		switch metric := metric.(type) {
 		case float64:

cmd/geth/usage.go

@@ -26,14 +26,14 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 // AppHelpTemplate is the test template for the default, global app help topic.
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2018 The go-ethereum Authors
+   Copyright 2013-2019 The go-ethereum Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -74,13 +74,18 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetworkIdFlag,
 			utils.TestnetFlag,
 			utils.RinkebyFlag,
+			utils.GoerliFlag,
 			utils.SyncModeFlag,
+			utils.ExitWhenSyncedFlag,
 			utils.GCModeFlag,
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
 			utils.LightServFlag,
+			utils.LightBandwidthInFlag,
+			utils.LightBandwidthOutFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
+			utils.WhitelistFlag,
 		},
 	},
 	{
@@ -132,8 +137,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
-			utils.TrieCacheGenFlag,
 		},
 	},
 	{
@@ -141,6 +146,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.UnlockedAccountFlag,
 			utils.PasswordFileFlag,
+			utils.ExternalSignerFlag,
 		},
 	},
 	{
@@ -207,6 +213,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
 			utils.VMEnableDebugFlag,
+			utils.EVMInterpreterFlag,
+			utils.EWASMInterpreterFlag,
 		},
 	},
 	{
@@ -225,7 +233,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.MetricsInfluxDBDatabaseFlag,
 			utils.MetricsInfluxDBUsernameFlag,
 			utils.MetricsInfluxDBPasswordFlag,
-			utils.MetricsInfluxDBHostTagFlag,
+			utils.MetricsInfluxDBTagsFlag,
 		},
 	},
 	{
@@ -301,7 +309,7 @@ func init() {
 					categorized[flag.String()] = struct{}{}
 				}
 			}
-			uncategorized := []cli.Flag{}
+			var uncategorized []cli.Flag
 			for _, flag := range data.(*cli.App).Flags {
 				if _, ok := categorized[flag.String()]; !ok {
 					if strings.HasPrefix(flag.GetName(), "dashboard") {

cmd/p2psim/main.go

@@ -47,7 +47,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -285,7 +285,7 @@ func createNode(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		config.ID = discover.PubkeyID(&privKey.PublicKey)
+		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
 	if services := ctx.String("services"); services != "" {

cmd/puppeth/genesis.go

@@ -20,35 +20,41 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/big"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	math2 "github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-// cppEthereumGenesisSpec represents the genesis specification format used by the
+// alethGenesisSpec represents the genesis specification format used by the
 // C++ Ethereum implementation.
-type cppEthereumGenesisSpec struct {
+type alethGenesisSpec struct {
 	SealEngine string `json:"sealEngine"`
 	Params     struct {
-		AccountStartNonce       hexutil.Uint64 `json:"accountStartNonce"`
-		HomesteadForkBlock      hexutil.Uint64 `json:"homesteadForkBlock"`
-		EIP150ForkBlock         hexutil.Uint64 `json:"EIP150ForkBlock"`
-		EIP158ForkBlock         hexutil.Uint64 `json:"EIP158ForkBlock"`
-		ByzantiumForkBlock      hexutil.Uint64 `json:"byzantiumForkBlock"`
-		ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"`
-		NetworkID               hexutil.Uint64 `json:"networkID"`
-		ChainID                 hexutil.Uint64 `json:"chainID"`
-		MaximumExtraDataSize    hexutil.Uint64 `json:"maximumExtraDataSize"`
-		MinGasLimit             hexutil.Uint64 `json:"minGasLimit"`
-		MaxGasLimit             hexutil.Uint64 `json:"maxGasLimit"`
-		GasLimitBoundDivisor    hexutil.Uint64 `json:"gasLimitBoundDivisor"`
-		MinimumDifficulty       *hexutil.Big   `json:"minimumDifficulty"`
-		DifficultyBoundDivisor  *hexutil.Big   `json:"difficultyBoundDivisor"`
-		DurationLimit           *hexutil.Big   `json:"durationLimit"`
-		BlockReward             *hexutil.Big   `json:"blockReward"`
+		AccountStartNonce       math2.HexOrDecimal64   `json:"accountStartNonce"`
+		MaximumExtraDataSize    hexutil.Uint64         `json:"maximumExtraDataSize"`
+		HomesteadForkBlock      hexutil.Uint64         `json:"homesteadForkBlock"`
+		DaoHardforkBlock        math2.HexOrDecimal64   `json:"daoHardforkBlock"`
+		EIP150ForkBlock         hexutil.Uint64         `json:"EIP150ForkBlock"`
+		EIP158ForkBlock         hexutil.Uint64         `json:"EIP158ForkBlock"`
+		ByzantiumForkBlock      hexutil.Uint64         `json:"byzantiumForkBlock"`
+		ConstantinopleForkBlock hexutil.Uint64         `json:"constantinopleForkBlock"`
+		MinGasLimit             hexutil.Uint64         `json:"minGasLimit"`
+		MaxGasLimit             hexutil.Uint64         `json:"maxGasLimit"`
+		TieBreakingGas          bool                   `json:"tieBreakingGas"`
+		GasLimitBoundDivisor    math2.HexOrDecimal64   `json:"gasLimitBoundDivisor"`
+		MinimumDifficulty       *hexutil.Big           `json:"minimumDifficulty"`
+		DifficultyBoundDivisor  *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"`
+		DurationLimit           *math2.HexOrDecimal256 `json:"durationLimit"`
+		BlockReward             *hexutil.Big           `json:"blockReward"`
+		NetworkID               hexutil.Uint64         `json:"networkID"`
+		ChainID                 hexutil.Uint64         `json:"chainID"`
+		AllowFutureBlocks       bool                   `json:"allowFutureBlocks"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"`
+	Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"`
 }
 
-// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled
+// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
-type cppEthereumGenesisSpecAccount struct {
-	Balance     *hexutil.Big                   `json:"balance"`
-	Nonce       uint64                         `json:"nonce,omitempty"`
-	Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"`
+type alethGenesisSpecAccount struct {
+	Balance     *math2.HexOrDecimal256   `json:"balance"`
+	Nonce       uint64                   `json:"nonce,omitempty"`
+	Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"`
 }
 
-// cppEthereumGenesisSpecBuiltin is the precompiled contract definition.
-type cppEthereumGenesisSpecBuiltin struct {
-	Name          string                               `json:"name,omitempty"`
-	StartingBlock hexutil.Uint64                       `json:"startingBlock,omitempty"`
-	Linear        *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"`
+// alethGenesisSpecBuiltin is the precompiled contract definition.
+type alethGenesisSpecBuiltin struct {
+	Name          string                         `json:"name,omitempty"`
+	StartingBlock hexutil.Uint64                 `json:"startingBlock,omitempty"`
+	Linear        *alethGenesisSpecLinearPricing `json:"linear,omitempty"`
 }
 
-type cppEthereumGenesisSpecLinearPricing struct {
+type alethGenesisSpecLinearPricing struct {
 	Base uint64 `json:"base"`
 	Word uint64 `json:"word"`
 }
 
-// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific
+// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific
 // chain specification format.
-func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) {
-	// Only ethash is currently supported between go-ethereum and cpp-ethereum
+func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) {
+	// Only ethash is currently supported between go-ethereum and aleth
 	if genesis.Config.Ethash == nil {
 		return nil, errors.New("unsupported consensus engine")
 	}
-	// Reconstruct the chain spec in Parity's format
-	spec := &cppEthereumGenesisSpec{
+	// Reconstruct the chain spec in Aleth format
+	spec := &alethGenesisSpec{
 		SealEngine: "Ethash",
 	}
+	// Some defaults
 	spec.Params.AccountStartNonce = 0
+	spec.Params.TieBreakingGas = false
+	spec.Params.AllowFutureBlocks = false
+	spec.Params.DaoHardforkBlock = 0
+
 	spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64())
 	spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64())
 	spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64())
-	spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())
-	spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64)
+
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
 
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
-
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64)
+	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64)
 	spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
-	spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
-	spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
+	spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
+	spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit)
 	spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
 
 	spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8))
@@ -126,77 +143,109 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &cppEthereumGenesisSpecAccount{
-			Balance: (*hexutil.Big)(account.Balance),
-			Nonce:   account.Nonce,
-		}
-	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3},
+		spec.setAccount(address, account)
 	}
+
+	spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover",
+		Linear: &alethGenesisSpecLinearPricing{Base: 3000}})
+	spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256",
+		Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}})
+	spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160",
+		Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}})
+	spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity",
+		Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-		}
-		spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-		}
+		spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
+		spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+			Linear:        &alethGenesisSpecLinearPricing{Base: 500}})
+		spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+			Linear:        &alethGenesisSpecLinearPricing{Base: 40000}})
+		spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
 	}
 	return spec, nil
 }
 
+func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+	addr := common.UnprefixedAddress(common.BytesToAddress([]byte{address}))
+	if _, exist := spec.Accounts[addr]; !exist {
+		spec.Accounts[addr] = &alethGenesisSpecAccount{}
+	}
+	spec.Accounts[addr].Precompiled = data
+}
+
+func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+
+	a, exist := spec.Accounts[common.UnprefixedAddress(address)]
+	if !exist {
+		a = &alethGenesisSpecAccount{}
+		spec.Accounts[common.UnprefixedAddress(address)] = a
+	}
+	a.Balance = (*math2.HexOrDecimal256)(account.Balance)
+	a.Nonce = account.Nonce
+
+}
+
+func (spec *alethGenesisSpec) setByzantium(num *big.Int) {
+	spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64())
+}
+
+func (spec *alethGenesisSpec) setConstantinople(num *big.Int) {
+	spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64())
+}
+
 // parityChainSpec is the chain specification format used by Parity.
 type parityChainSpec struct {
-	Name   string `json:"name"`
-	Engine struct {
+	Name    string `json:"name"`
+	Datadir string `json:"dataDir"`
+	Engine  struct {
 		Ethash struct {
 			Params struct {
-				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
-				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
-				DurationLimit          *hexutil.Big `json:"durationLimit"`
-				BlockReward            *hexutil.Big `json:"blockReward"`
-				HomesteadTransition    uint64       `json:"homesteadTransition"`
-				EIP150Transition       uint64       `json:"eip150Transition"`
-				EIP160Transition       uint64       `json:"eip160Transition"`
-				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
-				EIP161dTransition      uint64       `json:"eip161dTransition"`
-				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
-				EIP100bTransition      uint64       `json:"eip100bTransition"`
-				EIP649Transition       uint64       `json:"eip649Transition"`
+				MinimumDifficulty      *hexutil.Big      `json:"minimumDifficulty"`
+				DifficultyBoundDivisor *hexutil.Big      `json:"difficultyBoundDivisor"`
+				DurationLimit          *hexutil.Big      `json:"durationLimit"`
+				BlockReward            map[string]string `json:"blockReward"`
+				DifficultyBombDelays   map[string]string `json:"difficultyBombDelays"`
+				HomesteadTransition    hexutil.Uint64    `json:"homesteadTransition"`
+				EIP100bTransition      hexutil.Uint64    `json:"eip100bTransition"`
 			} `json:"params"`
 		} `json:"Ethash"`
 	} `json:"engine"`
 
 	Params struct {
-		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
-		MinGasLimit          hexutil.Uint64 `json:"minGasLimit"`
-		GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"`
-		NetworkID            hexutil.Uint64 `json:"networkID"`
-		MaxCodeSize          uint64         `json:"maxCodeSize"`
-		EIP155Transition     uint64         `json:"eip155Transition"`
-		EIP98Transition      uint64         `json:"eip98Transition"`
-		EIP86Transition      uint64         `json:"eip86Transition"`
-		EIP140Transition     uint64         `json:"eip140Transition"`
-		EIP211Transition     uint64         `json:"eip211Transition"`
-		EIP214Transition     uint64         `json:"eip214Transition"`
-		EIP658Transition     uint64         `json:"eip658Transition"`
+		AccountStartNonce        hexutil.Uint64       `json:"accountStartNonce"`
+		MaximumExtraDataSize     hexutil.Uint64       `json:"maximumExtraDataSize"`
+		MinGasLimit              hexutil.Uint64       `json:"minGasLimit"`
+		GasLimitBoundDivisor     math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"`
+		NetworkID                hexutil.Uint64       `json:"networkID"`
+		ChainID                  hexutil.Uint64       `json:"chainID"`
+		MaxCodeSize              hexutil.Uint64       `json:"maxCodeSize"`
+		MaxCodeSizeTransition    hexutil.Uint64       `json:"maxCodeSizeTransition"`
+		EIP98Transition          hexutil.Uint64       `json:"eip98Transition"`
+		EIP150Transition         hexutil.Uint64       `json:"eip150Transition"`
+		EIP160Transition         hexutil.Uint64       `json:"eip160Transition"`
+		EIP161abcTransition      hexutil.Uint64       `json:"eip161abcTransition"`
+		EIP161dTransition        hexutil.Uint64       `json:"eip161dTransition"`
+		EIP155Transition         hexutil.Uint64       `json:"eip155Transition"`
+		EIP140Transition         hexutil.Uint64       `json:"eip140Transition"`
+		EIP211Transition         hexutil.Uint64       `json:"eip211Transition"`
+		EIP214Transition         hexutil.Uint64       `json:"eip214Transition"`
+		EIP658Transition         hexutil.Uint64       `json:"eip658Transition"`
+		EIP145Transition         hexutil.Uint64       `json:"eip145Transition"`
+		EIP1014Transition        hexutil.Uint64       `json:"eip1014Transition"`
+		EIP1052Transition        hexutil.Uint64       `json:"eip1052Transition"`
+		EIP1283Transition        hexutil.Uint64       `json:"eip1283Transition"`
+		EIP1283DisableTransition hexutil.Uint64       `json:"eip1283DisableTransition"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -215,22 +264,22 @@ type parityChainSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Nodes    []string                                   `json:"nodes"`
-	Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"`
+	Nodes    []string                                             `json:"nodes"`
+	Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"`
 }
 
 // parityChainSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
 type parityChainSpecAccount struct {
-	Balance *hexutil.Big            `json:"balance"`
-	Nonce   uint64                  `json:"nonce,omitempty"`
+	Balance math2.HexOrDecimal256   `json:"balance"`
+	Nonce   math2.HexOrDecimal64    `json:"nonce,omitempty"`
 	Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"`
 }
 
 // parityChainSpecBuiltin is the precompiled contract definition.
 type parityChainSpecBuiltin struct {
 	Name       string                  `json:"name,omitempty"`
-	ActivateAt uint64                  `json:"activate_at,omitempty"`
+	ActivateAt math2.HexOrDecimal64    `json:"activate_at,omitempty"`
 	Pricing    *parityChainSpecPricing `json:"pricing,omitempty"`
 }
 
@@ -265,34 +314,56 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	}
 	// Reconstruct the chain spec in Parity's format
 	spec := &parityChainSpec{
-		Name:  network,
-		Nodes: bootnodes,
+		Name:    network,
+		Nodes:   bootnodes,
+		Datadir: strings.ToLower(network),
 	}
+	spec.Engine.Ethash.Params.BlockReward = make(map[string]string)
+	spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string)
+	// Frontier
 	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
 	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
 	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
-	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
-	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64()
-	spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward)
-	spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward)
+
+	// Homestead
+	spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64())
+
+	// Tangerine Whistle : 150
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md
+	spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64())
+
+	// Spurious Dragon: 155, 160, 161, 170
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md
+	spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+	spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
+	// ConstantinopleFix (remove eip-1283)
+	if num := genesis.Config.PetersburgBlock; num != nil {
+		spec.setConstantinopleFix(num)
+	}
 
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
+	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.MaxCodeSize = params.MaxCodeSize
-	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()
-	spec.Params.EIP98Transition = math.MaxUint64
-	spec.Params.EIP86Transition = math.MaxUint64
-	spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64()
+	// geth has it set from zero
+	spec.Params.MaxCodeSizeTransition = 0
+
+	// Disable this one
+	spec.Params.EIP98Transition = math.MaxInt64
 
 	spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8))
 	binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce)
@@ -305,42 +376,81 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*parityChainSpecAccount)
+	spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &parityChainSpecAccount{
-			Balance: (*hexutil.Big)(account.Balance),
-			Nonce:   account.Nonce,
+		bal := math2.HexOrDecimal256(*account.Balance)
+
+		spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{
+			Balance: bal,
+			Nonce:   math2.HexOrDecimal64(account.Nonce),
 		}
 	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{
-		Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover",
+		Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}})
+
+	spec.setPrecompile(2, &parityChainSpecBuiltin{
 		Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{
+	})
+	spec.setPrecompile(3, &parityChainSpecBuiltin{
 		Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{
+	})
+	spec.setPrecompile(4, &parityChainSpecBuiltin{
 		Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}},
-	}
+	})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{
-			Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
-		}
+		blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64())
+		spec.setPrecompile(5, &parityChainSpecBuiltin{
+			Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
+		})
+		spec.setPrecompile(6, &parityChainSpecBuiltin{
+			Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
+		})
+		spec.setPrecompile(7, &parityChainSpecBuiltin{
+			Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
+		})
+		spec.setPrecompile(8, &parityChainSpecBuiltin{
+			Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
+		})
 	}
 	return spec, nil
 }
 
+func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
+	}
+	a := common.UnprefixedAddress(common.BytesToAddress([]byte{address}))
+	if _, exist := spec.Accounts[a]; !exist {
+		spec.Accounts[a] = &parityChainSpecAccount{}
+	}
+	spec.Accounts[a].Builtin = data
+}
+
+func (spec *parityChainSpec) setByzantium(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Engine.Ethash.Params.EIP100bTransition = n
+	spec.Params.EIP140Transition = n
+	spec.Params.EIP211Transition = n
+	spec.Params.EIP214Transition = n
+	spec.Params.EIP658Transition = n
+}
+
+func (spec *parityChainSpec) setConstantinople(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Params.EIP145Transition = n
+	spec.Params.EIP1014Transition = n
+	spec.Params.EIP1052Transition = n
+	spec.Params.EIP1283Transition = n
+}
+
+func (spec *parityChainSpec) setConstantinopleFix(num *big.Int) {
+	spec.Params.EIP1283DisableTransition = hexutil.Uint64(num.Uint64())
+}
+
 // pyEthereumGenesisSpec represents the genesis specification format used by the
 // Python Ethereum implementation.
 type pyEthereumGenesisSpec struct {

cmd/puppeth/genesis_test.go

@@ -0,0 +1,109 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/core"
+)
+
+// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet.
+func TestAlethSturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newAlethGenesisSpec("stureby", &genesis)
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &alethGenesisSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}
+
+// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet.
+func TestParitySturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newParityChainSpec("Stureby", &genesis, []string{})
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &parityChainSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	expspec.Nodes = []string{}
+
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}