params: v1.5.8-stable

les: fix private net issues, enable adding peers manually again

.travis.yml

@@ -65,10 +65,11 @@ matrix:
 
         # Build the Android archive and upload it to Maven Central and Azure
         - brew update
-        - brew install android-sdk maven gpg
+        - travis_wait 60 brew install android-sdk android-ndk maven gpg
         - alias gpg="gpg2"
 
         - export ANDROID_HOME=/usr/local/opt/android-sdk
+        - export ANDROID_NDK=/usr/local/opt/android-ndk
         - echo "y" | android update sdk --no-ui --filter `android list sdk | grep "SDK Platform Android" | grep -E 'API 15|API 19|API 24' | awk '{print $1}' | cut -d '-' -f 1 | tr '\n' ','`
 
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds

VERSION

@@ -1 +1 @@
-1.5.7
+1.5.8

accounts/abi/bind/backends/simulated.go

@@ -61,7 +61,7 @@ type SimulatedBackend struct {
 func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
 	core.WriteGenesisBlockForTesting(database, accounts...)
-	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux))
+	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain}
 	backend.rollback()
 	return backend
@@ -201,10 +201,32 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error
 func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (*big.Int, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot())
 
-	_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
-	return gas, err
+	// Binary search the gas requirement, as it may be higher than the amount used
+	var lo, hi uint64
+	if call.Gas != nil {
+		hi = call.Gas.Uint64()
+	} else {
+		hi = b.pendingBlock.GasLimit().Uint64()
+	}
+	for lo+1 < hi {
+		// Take a guess at the gas, and check transaction validity
+		mid := (hi + lo) / 2
+		call.Gas = new(big.Int).SetUint64(mid)
+
+		snapshot := b.pendingState.Snapshot()
+		_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+		b.pendingState.RevertToSnapshot(snapshot)
+
+		// If the transaction became invalid or used all the gas (failed), raise the gas limit
+		if err != nil || gas.Cmp(call.Gas) == 0 {
+			lo = mid
+			continue
+		}
+		// Otherwise assume the transaction succeeded, lower the gas limit
+		hi = mid
+	}
+	return new(big.Int).SetUint64(hi), nil
 }
 
 // callContract implemens common code between normal and pending contract calls.

accounts/abi/bind/bind_test.go

@@ -341,11 +341,11 @@ var bindTests = []struct {
 	{
 		`NonExistent`,
 		`
-		contract NonExistent {
-			function String() constant returns(string) {
-				return "I don't exist";
+			contract NonExistent {
+				function String() constant returns(string) {
+					return "I don't exist";
+				}
 			}
-		}
 		`,
 		`6060604052609f8060106000396000f3606060405260e060020a6000350463f97a60058114601a575b005b600060605260c0604052600d60809081527f4920646f6e27742065786973740000000000000000000000000000000000000060a052602060c0908152600d60e081905281906101009060a09080838184600060046012f15050815172ffffffffffffffffffffffffffffffffffffff1916909152505060405161012081900392509050f3`,
 		`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
@@ -365,6 +365,49 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Tests that gas estimation works for contracts with	weird gas mechanics too.
+	{
+		`FunkyGasPattern`,
+		`
+			contract FunkyGasPattern {
+				string public field;
+
+				function SetField(string value) {
+					// This check will screw gas estimation! Good, good!
+					if (msg.gas < 100000) {
+						throw;
+					}
+					field = value;
+				}
+			}
+		`,
+		`606060405261021c806100126000396000f3606060405260e060020a600035046323fcf32a81146100265780634f28bf0e1461007b575b005b6040805160206004803580820135601f8101849004840285018401909552848452610024949193602493909291840191908190840183828082843750949650505050505050620186a05a101561014e57610002565b6100db60008054604080516020601f600260001961010060018816150201909516949094049384018190048102820181019092528281529291908301828280156102145780601f106101e957610100808354040283529160200191610214565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561013b5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b505050565b8060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106101b557805160ff19168380011785555b506101499291505b808211156101e557600081556001016101a1565b82800160010185558215610199579182015b828111156101995782518260005055916020019190600101906101c7565b5090565b820191906000526020600020905b8154815290600101906020018083116101f757829003601f168201915b50505050508156`,
+		`[{"constant":false,"inputs":[{"name":"value","type":"string"}],"name":"SetField","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"field","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+
+			// Deploy a funky gas pattern contract
+			_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy funky contract: %v", err)
+			}
+			sim.Commit()
+
+			// Set the field with automatic estimation and check that it succeeds
+			auth.GasLimit = nil
+			if _, err := limiter.SetField(auth, "automatic"); err != nil {
+				t.Fatalf("Failed to call automatically gased transaction: %v", err)
+			}
+			sim.Commit()
+
+			if field, _ := limiter.Field(nil); field != "automatic" {
+				t.Fatalf("Field mismatch: have %v, want %v", field, "automatic")
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and

accounts/account_manager.go

@@ -113,9 +113,9 @@ func (am *Manager) Accounts() []Account {
 	return am.cache.accounts()
 }
 
-// DeleteAccount deletes the key matched by account if the passphrase is correct.
-// If a contains no filename, the address must match a unique key.
-func (am *Manager) DeleteAccount(a Account, passphrase string) error {
+// Delete deletes the key matched by account if the passphrase is correct.
+// If the account contains no filename, the address must match a unique key.
+func (am *Manager) Delete(a Account, passphrase string) error {
 	// Decrypting the key isn't really necessary, but we do
 	// it anyway to check the password and zero out the key
 	// immediately afterwards.

accounts/accounts_test.go

@@ -53,14 +53,14 @@ func TestManager(t *testing.T) {
 	if err := am.Update(a, "foo", "bar"); err != nil {
 		t.Errorf("Update error: %v", err)
 	}
-	if err := am.DeleteAccount(a, "bar"); err != nil {
-		t.Errorf("DeleteAccount error: %v", err)
+	if err := am.Delete(a, "bar"); err != nil {
+		t.Errorf("Delete error: %v", err)
 	}
 	if common.FileExist(a.File) {
-		t.Errorf("account file %s should be gone after DeleteAccount", a.File)
+		t.Errorf("account file %s should be gone after Delete", a.File)
 	}
 	if am.HasAddress(a.Address) {
-		t.Errorf("HasAccount(%x) should've returned true after DeleteAccount", a.Address)
+		t.Errorf("HasAccount(%x) should've returned true after Delete", a.Address)
 	}
 }
 

build/ci.go

@@ -309,7 +309,7 @@ func spellcheck(packages []string) {
 	// Ensure the spellchecker is available
 	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
 
-	// Windows chokes on long argument lists, check packages individualy
+	// Windows chokes on long argument lists, check packages individually
 	for _, pkg := range packages {
 		// The spell checker doesn't work on packages, gather all .go files for it
 		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
@@ -700,9 +700,16 @@ func doAndroidArchive(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
+	// Sanity check that the SDK and NDK are installed and set
+	if os.Getenv("ANDROID_HOME") == "" {
+		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
+	}
+	if os.Getenv("ANDROID_NDK") == "" {
+		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
+	}
 	// Build the Android archive and Maven resources
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
-	build.MustRun(gomobileTool("init"))
+	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {

cmd/evm/main.go

@@ -161,6 +161,7 @@ func run(ctx *cli.Context) error {
 			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
 			EVMConfig: vm.Config{
 				Tracer:             logger,
+				Debug:              ctx.GlobalBool(DebugFlag.Name),
 				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
 			},
 		})
@@ -176,6 +177,7 @@ func run(ctx *cli.Context) error {
 			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
 			EVMConfig: vm.Config{
 				Tracer:             logger,
+				Debug:              ctx.GlobalBool(DebugFlag.Name),
 				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
 			},
 		})

cmd/geth/chaincmd.go

@@ -123,6 +123,7 @@ func initGenesis(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("failed to read genesis file: %v", err)
 	}
+	defer genesisFile.Close()
 
 	block, err := core.WriteGenesisBlock(chaindb, genesisFile)
 	if err != nil {

cmd/geth/main.go

@@ -106,7 +106,6 @@ func init() {
 		utils.AutoDAGFlag,
 		utils.TargetGasLimitFlag,
 		utils.NATFlag,
-		utils.NatspecEnabledFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,
 		utils.NetrestrictFlag,
@@ -132,6 +131,7 @@ func init() {
 		utils.VMForceJitFlag,
 		utils.VMJitCacheFlag,
 		utils.VMEnableJitFlag,
+		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.RPCCORSDomainFlag,
 		utils.EthStatsURLFlag,

cmd/geth/usage.go

@@ -155,6 +155,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.VMEnableJitFlag,
 			utils.VMForceJitFlag,
 			utils.VMJitCacheFlag,
+			utils.VMEnableDebugFlag,
 		},
 	},
 	{
@@ -169,7 +170,6 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
-			utils.NatspecEnabledFlag,
 		},
 	},
 	{

cmd/swarm/hash.go

@@ -36,6 +36,7 @@ func hash(ctx *cli.Context) {
 		fmt.Println("Error opening file " + args[1])
 		os.Exit(1)
 	}
+	defer f.Close()
 
 	stat, _ := f.Stat()
 	chunker := storage.NewTreeChunker(storage.NewChunkerParams())

cmd/swarm/main.go

@@ -39,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/network"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -76,7 +75,6 @@ var (
 	SwarmNetworkIdFlag = cli.IntFlag{
 		Name:  "bzznetworkid",
 		Usage: "Network identifier (integer, default 3=swarm testnet)",
-		Value: network.NetworkId,
 	}
 	SwarmConfigPathFlag = cli.StringFlag{
 		Name:  "bzzconfig",
@@ -242,6 +240,7 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
+
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -249,6 +248,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if bzzdir == "" {
 		bzzdir = stack.InstanceDir()
 	}
+
 	bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name))
 	if err != nil {
 		utils.Fatalf("unable to configure swarm: %v", err)
@@ -280,6 +280,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 
 func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	keyid := ctx.GlobalString(SwarmAccountFlag.Name)
+
 	if keyid == "" {
 		utils.Fatalf("Option %q is required", SwarmAccountFlag.Name)
 	}

cmd/utils/flags.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -130,10 +131,6 @@ var (
 		Name:  "identity",
 		Usage: "Custom node name",
 	}
-	NatspecEnabledFlag = cli.BoolFlag{
-		Name:  "natspec",
-		Usage: "Enable NatSpec confirmation notice",
-	}
 	DocRootFlag = DirectoryFlag{
 		Name:  "docroot",
 		Usage: "Document Root for HTTPClient file scheme",
@@ -230,6 +227,10 @@ var (
 		Name:  "jitvm",
 		Usage: "Enable the JIT VM",
 	}
+	VMEnableDebugFlag = cli.BoolFlag{
+		Name:  "vmdebug",
+		Usage: "Record information useful for VM and contract debugging",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -653,6 +654,10 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
 		vsn += "-" + gitCommit[:8]
 	}
 
+	// if we're running a light client or server, force enable the v5 peer discovery unless it is explicitly disabled with --nodiscover
+	// note that explicitly specifying --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
+	forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name)
+
 	config := &node.Config{
 		DataDir:           MakeDataDir(ctx),
 		KeyStoreDir:       ctx.GlobalString(KeyStoreDirFlag.Name),
@@ -661,8 +666,8 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
 		Name:              name,
 		Version:           vsn,
 		UserIdent:         makeNodeUserIdent(ctx),
-		NoDiscovery:       ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name),
-		DiscoveryV5:       ctx.GlobalBool(DiscoveryV5Flag.Name) || ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0,
+		NoDiscovery:       ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name), // always disable v4 discovery in light client mode
+		DiscoveryV5:       ctx.GlobalBool(DiscoveryV5Flag.Name) || forceV5Discovery,
 		DiscoveryV5Addr:   MakeDiscoveryV5Address(ctx),
 		BootstrapNodes:    MakeBootstrapNodes(ctx),
 		BootstrapNodesV5:  MakeBootstrapNodesV5(ctx),
@@ -730,7 +735,6 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),
 		MinerThreads:            ctx.GlobalInt(MinerThreadsFlag.Name),
 		ExtraData:               MakeMinerExtra(extra, ctx),
-		NatSpec:                 ctx.GlobalBool(NatspecEnabledFlag.Name),
 		DocRoot:                 ctx.GlobalString(DocRootFlag.Name),
 		GasPrice:                common.String2Big(ctx.GlobalString(GasPriceFlag.Name)),
 		GpoMinGasPrice:          common.String2Big(ctx.GlobalString(GpoMinGasPriceFlag.Name)),
@@ -741,6 +745,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		GpobaseCorrectionFactor: ctx.GlobalInt(GpobaseCorrectionFactorFlag.Name),
 		SolcPath:                ctx.GlobalString(SolcPathFlag.Name),
 		AutoDAG:                 ctx.GlobalBool(AutoDAGFlag.Name) || ctx.GlobalBool(MiningEnabledFlag.Name),
+		EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name),
 	}
 
 	// Override any default configs in dev mode or the test net
@@ -912,7 +917,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	if !ctx.GlobalBool(FakePoWFlag.Name) {
 		pow = ethash.New()
 	}
-	chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux))
+	chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux), vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)})
 	if err != nil {
 		Fatalf("Could not start chainmanager: %v", err)
 	}

cmd/wnode/main.go

@@ -0,0 +1,537 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// This is a simple Whisper node. It could be used as a stand-alone bootstrap node.
+// Also, could be used for different test and diagnostics purposes.
+
+package main
+
+import (
+	"bufio"
+	"crypto/ecdsa"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/nat"
+	"github.com/ethereum/go-ethereum/whisper/mailserver"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	"golang.org/x/crypto/pbkdf2"
+)
+
+const quitCommand = "~Q"
+
+// singletons
+var (
+	server     *p2p.Server
+	shh        *whisper.Whisper
+	done       chan struct{}
+	mailServer mailserver.WMailServer
+
+	input = bufio.NewReader(os.Stdin)
+)
+
+// encryption
+var (
+	symKey     []byte
+	pub        *ecdsa.PublicKey
+	asymKey    *ecdsa.PrivateKey
+	nodeid     *ecdsa.PrivateKey
+	topic      whisper.TopicType
+	filterID   uint32
+	msPassword string
+)
+
+// cmd arguments
+var (
+	echoMode       = flag.Bool("e", false, "echo mode: prints some arguments for diagnostics")
+	bootstrapMode  = flag.Bool("b", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	forwarderMode  = flag.Bool("f", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	mailServerMode = flag.Bool("s", false, "mail server mode: delivers expired messages on demand")
+	requestMail    = flag.Bool("r", false, "request expired messages from the bootstrap server")
+	asymmetricMode = flag.Bool("a", false, "use asymmetric encryption")
+	testMode       = flag.Bool("t", false, "use of predefined parameters for diagnostics")
+	generateKey    = flag.Bool("k", false, "generate and show the private key")
+
+	argTTL       = flag.Uint("ttl", 30, "time-to-live for messages in seconds")
+	argWorkTime  = flag.Uint("work", 5, "work time in seconds")
+	argPoW       = flag.Float64("pow", whisper.MinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argServerPoW = flag.Float64("mspow", whisper.MinimumPoW, "PoW requirement for Mail Server request")
+
+	argIP     = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:30303)")
+	argSalt   = flag.String("salt", "", "salt (for topic and key derivation)")
+	argPub    = flag.String("pub", "", "public key for asymmetric encryption")
+	argDBPath = flag.String("dbpath", "", "path to the server's DB directory")
+	argIDFile = flag.String("idfile", "", "file name with node id (private key)")
+	argEnode  = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
+	argTopic  = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+)
+
+func main() {
+	processArgs()
+	initialize()
+	run()
+}
+
+func processArgs() {
+	flag.Parse()
+
+	if len(*argIDFile) > 0 {
+		var err error
+		nodeid, err = crypto.LoadECDSA(*argIDFile)
+		if err != nil {
+			utils.Fatalf("Failed to load file [%s]: %s.", *argIDFile, err)
+		}
+	}
+
+	const enodePrefix = "enode://"
+	if len(*argEnode) > 0 {
+		if (*argEnode)[:len(enodePrefix)] != enodePrefix {
+			*argEnode = enodePrefix + *argEnode
+		}
+	}
+
+	if len(*argTopic) > 0 {
+		x, err := hex.DecodeString(*argTopic)
+		if err != nil {
+			utils.Fatalf("Failed to parse the topic: %s", err)
+		}
+		topic = whisper.BytesToTopic(x)
+	}
+
+	if *asymmetricMode && len(*argPub) > 0 {
+		pub = crypto.ToECDSAPub(common.FromHex(*argPub))
+		if !isKeyValid(pub) {
+			utils.Fatalf("invalid public key")
+		}
+	}
+
+	if *echoMode {
+		echo()
+	}
+}
+
+func echo() {
+	fmt.Printf("ttl = %d \n", *argTTL)
+	fmt.Printf("workTime = %d \n", *argWorkTime)
+	fmt.Printf("pow = %f \n", *argPoW)
+	fmt.Printf("mspow = %f \n", *argServerPoW)
+	fmt.Printf("ip = %s \n", *argIP)
+	fmt.Printf("salt = %s \n", *argSalt)
+	fmt.Printf("pub = %s \n", common.ToHex(crypto.FromECDSAPub(pub)))
+	fmt.Printf("idfile = %s \n", *argIDFile)
+	fmt.Printf("dbpath = %s \n", *argDBPath)
+	fmt.Printf("boot = %s \n", *argEnode)
+}
+
+func initialize() {
+	glog.SetV(logger.Warn)
+	glog.SetToStderr(true)
+
+	done = make(chan struct{})
+	var peers []*discover.Node
+	var err error
+
+	if *generateKey {
+		key, err := crypto.GenerateKey()
+		if err != nil {
+			utils.Fatalf("Failed to generate private key: %s", err)
+		}
+		k := hex.EncodeToString(crypto.FromECDSA(key))
+		fmt.Printf("Random private key: %s \n", k)
+		os.Exit(0)
+	}
+
+	if *testMode {
+		password := []byte("test password for symmetric encryption")
+		salt := []byte("test salt for symmetric encryption")
+		symKey = pbkdf2.Key(password, salt, 64, 32, sha256.New)
+		topic = whisper.TopicType{0xFF, 0xFF, 0xFF, 0xFF}
+		msPassword = "mail server test password"
+	}
+
+	if *bootstrapMode {
+		if len(*argIP) == 0 {
+			argIP = scanLineA("Please enter your IP and port (e.g. 127.0.0.1:30348): ")
+		}
+	} else {
+		if len(*argEnode) == 0 {
+			argEnode = scanLineA("Please enter the peer's enode: ")
+		}
+		peer := discover.MustParseNode(*argEnode)
+		peers = append(peers, peer)
+	}
+
+	if *mailServerMode {
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+		shh = whisper.NewWhisper(&mailServer)
+		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
+	} else {
+		shh = whisper.NewWhisper(nil)
+	}
+
+	asymKey = shh.NewIdentity()
+	if nodeid == nil {
+		nodeid = shh.NewIdentity()
+	}
+
+	server = &p2p.Server{
+		Config: p2p.Config{
+			PrivateKey:     nodeid,
+			MaxPeers:       128,
+			Name:           common.MakeName("whisper-go", "5.0"),
+			Protocols:      shh.Protocols(),
+			ListenAddr:     *argIP,
+			NAT:            nat.Any(),
+			BootstrapNodes: peers,
+			StaticNodes:    peers,
+			TrustedNodes:   peers,
+		},
+	}
+}
+
+func startServer() {
+	err := server.Start()
+	if err != nil {
+		utils.Fatalf("Failed to start Whisper peer: %s.", err)
+	}
+
+	fmt.Printf("my public key: %s \n", common.ToHex(crypto.FromECDSAPub(&asymKey.PublicKey)))
+	fmt.Println(server.NodeInfo().Enode)
+
+	if *bootstrapMode {
+		configureNode()
+		fmt.Println("Bootstrap Whisper node started")
+	} else {
+		fmt.Println("Whisper node started")
+		// first see if we can establish connection, then ask for user input
+		waitForConnection(true)
+		configureNode()
+	}
+
+	if !*forwarderMode {
+		fmt.Printf("Please type the message. To quit type: '%s'\n", quitCommand)
+	}
+}
+
+func isKeyValid(k *ecdsa.PublicKey) bool {
+	return k.X != nil && k.Y != nil
+}
+
+func configureNode() {
+	var err error
+	var p2pAccept bool
+
+	if *forwarderMode {
+		return
+	}
+
+	if *asymmetricMode {
+		if len(*argPub) == 0 {
+			s := scanLine("Please enter the peer's public key: ")
+			pub = crypto.ToECDSAPub(common.FromHex(s))
+			if !isKeyValid(pub) {
+				utils.Fatalf("Error: invalid public key")
+			}
+		}
+	}
+
+	if *requestMail {
+		p2pAccept = true
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+	}
+
+	if !*asymmetricMode && !*forwarderMode && !*testMode {
+		pass, err := console.Stdin.PromptPassword("Please enter the password: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase: %v", err)
+		}
+
+		if len(*argSalt) == 0 {
+			argSalt = scanLineA("Please enter the salt: ")
+		}
+
+		symKey = pbkdf2.Key([]byte(pass), []byte(*argSalt), 65356, 32, sha256.New)
+
+		if len(*argTopic) == 0 {
+			generateTopic([]byte(pass), []byte(*argSalt))
+		}
+	}
+
+	if *mailServerMode {
+		if len(*argDBPath) == 0 {
+			argDBPath = scanLineA("Please enter the path to DB file: ")
+		}
+	}
+
+	filter := whisper.Filter{
+		KeySym:    symKey,
+		KeyAsym:   asymKey,
+		Topics:    []whisper.TopicType{topic},
+		AcceptP2P: p2pAccept,
+	}
+	filterID = shh.Watch(&filter)
+	fmt.Printf("Filter is configured for the topic: %x \n", topic)
+}
+
+func generateTopic(password, salt []byte) {
+	const rounds = 4000
+	const size = 128
+	x1 := pbkdf2.Key(password, salt, rounds, size, sha512.New)
+	x2 := pbkdf2.Key(password, salt, rounds, size, sha1.New)
+	x3 := pbkdf2.Key(x1, x2, rounds, size, sha256.New)
+
+	for i := 0; i < size; i++ {
+		topic[i%whisper.TopicLength] ^= x3[i]
+	}
+}
+
+func waitForConnection(timeout bool) {
+	var cnt int
+	var connected bool
+	for !connected {
+		time.Sleep(time.Millisecond * 50)
+		connected = server.PeerCount() > 0
+		if timeout {
+			cnt++
+			if cnt > 1000 {
+				utils.Fatalf("Timeout expired, failed to connect")
+			}
+		}
+	}
+
+	fmt.Println("Connected to peer.")
+}
+
+func run() {
+	defer mailServer.Close()
+	startServer()
+	defer server.Stop()
+	shh.Start(nil)
+	defer shh.Stop()
+
+	if !*forwarderMode {
+		go messageLoop()
+	}
+
+	if *requestMail {
+		requestExpiredMessagesLoop()
+	} else {
+		sendLoop()
+	}
+}
+
+func sendLoop() {
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			close(done)
+			break
+		}
+		sendMsg([]byte(s))
+
+		if *asymmetricMode {
+			// print your own message for convenience,
+			// because in asymmetric mode it is impossible to decrypt it
+			hour, min, sec := time.Now().Clock()
+			from := crypto.PubkeyToAddress(asymKey.PublicKey)
+			fmt.Printf("\n%02d:%02d:%02d <%x>: %s\n", hour, min, sec, from, s)
+		}
+	}
+}
+
+func scanLine(prompt string) string {
+	if len(prompt) > 0 {
+		fmt.Print(prompt)
+	}
+	txt, err := input.ReadString('\n')
+	if err != nil {
+		utils.Fatalf("input error: %s", err)
+	}
+	txt = strings.TrimRight(txt, "\n\r")
+	return txt
+}
+
+func scanLineA(prompt string) *string {
+	s := scanLine(prompt)
+	return &s
+}
+
+func scanUint(prompt string) uint32 {
+	s := scanLine(prompt)
+	i, err := strconv.Atoi(s)
+	if err != nil {
+		utils.Fatalf("Fail to parse the lower time limit: %s", err)
+	}
+	return uint32(i)
+}
+
+func sendMsg(payload []byte) {
+	params := whisper.MessageParams{
+		Src:      asymKey,
+		Dst:      pub,
+		KeySym:   symKey,
+		Payload:  payload,
+		Topic:    topic,
+		TTL:      uint32(*argTTL),
+		PoW:      *argPoW,
+		WorkTime: uint32(*argWorkTime),
+	}
+
+	msg := whisper.NewSentMessage(&params)
+	envelope, err := msg.Wrap(&params)
+	if err != nil {
+		fmt.Printf("failed to seal message: %v \n", err)
+		return
+	}
+
+	err = shh.Send(envelope)
+	if err != nil {
+		fmt.Printf("failed to send message: %v \n", err)
+	}
+}
+
+func messageLoop() {
+	f := shh.GetFilter(filterID)
+	if f == nil {
+		utils.Fatalf("filter is not installed")
+	}
+
+	ticker := time.NewTicker(time.Millisecond * 50)
+
+	for {
+		select {
+		case <-ticker.C:
+			messages := f.Retrieve()
+			for _, msg := range messages {
+				printMessageInfo(msg)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+func printMessageInfo(msg *whisper.ReceivedMessage) {
+	timestamp := fmt.Sprintf("%d", msg.Sent) // unix timestamp for diagnostics
+	text := string(msg.Payload)
+
+	var address common.Address
+	if msg.Src != nil {
+		address = crypto.PubkeyToAddress(*msg.Src)
+	}
+
+	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+		fmt.Printf("\n%s <%x>: %s\n", timestamp, address, text) // message from myself
+	} else {
+		fmt.Printf("\n%s [%x]: %s\n", timestamp, address, text) // message from a peer
+	}
+}
+
+func requestExpiredMessagesLoop() {
+	var key, peerID []byte
+	var timeLow, timeUpp uint32
+	var t string
+	var xt, empty whisper.TopicType
+
+	err := shh.AddSymKey(mailserver.MailServerKeyName, []byte(msPassword))
+	if err != nil {
+		utils.Fatalf("Failed to create symmetric key for mail request: %s", err)
+	}
+	key = shh.GetSymKey(mailserver.MailServerKeyName)
+	peerID = extractIdFromEnode(*argEnode)
+	shh.MarkPeerTrusted(peerID)
+
+	for {
+		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
+		timeUpp = scanUint("Please enter the upper limit of the time range (unix timestamp): ")
+		t = scanLine("Please enter the topic (hexadecimal): ")
+		if len(t) >= whisper.TopicLength*2 {
+			x, err := hex.DecodeString(t)
+			if err != nil {
+				utils.Fatalf("Failed to parse the topic: %s", err)
+			}
+			xt = whisper.BytesToTopic(x)
+		}
+		if timeUpp == 0 {
+			timeUpp = 0xFFFFFFFF
+		}
+
+		data := make([]byte, 8+whisper.TopicLength)
+		binary.BigEndian.PutUint32(data, timeLow)
+		binary.BigEndian.PutUint32(data[4:], timeUpp)
+		copy(data[8:], xt[:])
+		if xt == empty {
+			data = data[:8]
+		}
+
+		var params whisper.MessageParams
+		params.PoW = *argServerPoW
+		params.Payload = data
+		params.KeySym = key
+		params.Src = nodeid
+		params.WorkTime = 5
+
+		msg := whisper.NewSentMessage(&params)
+		env, err := msg.Wrap(&params)
+		if err != nil {
+			utils.Fatalf("Wrap failed: %s", err)
+		}
+
+		err = shh.RequestHistoricMessages(peerID, env)
+		if err != nil {
+			utils.Fatalf("Failed to send P2P message: %s", err)
+		}
+
+		time.Sleep(time.Second * 5)
+	}
+}
+
+func extractIdFromEnode(s string) []byte {
+	n, err := discover.ParseNode(s)
+	if err != nil {
+		utils.Fatalf("Failed to parse enode: %s", err)
+		return nil
+	}
+	return n.ID[:]
+}

common/compiler/solidity_test.go

@@ -36,7 +36,7 @@ contract test {
    }
 }
 `
-	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --natspec-user file --natspec-dev file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
+	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
 )
 
 func skipWithoutSolc(t *testing.T) {
@@ -99,7 +99,7 @@ func TestSaveInfo(t *testing.T) {
 	if string(got) != testInfo {
 		t.Errorf("incorrect info.json extracted, expected:\n%s\ngot\n%s", testInfo, string(got))
 	}
-	wantHash := common.HexToHash("0x9f3803735e7f16120c5a140ab3f02121fd3533a9655c69b33a10e78752cc49b0")
+	wantHash := common.HexToHash("0x22450a77f0c3ff7a395948d07bc1456881226a1b6325f4189cb5f1254a824080")
 	if cinfohash != wantHash {
 		t.Errorf("content hash for info is incorrect. expected %v, got %v", wantHash.Hex(), cinfohash.Hex())
 	}

console/console.go

@@ -137,10 +137,14 @@ func (c *Console) init(preload []string) error {
 			continue // manually mapped or ignore
 		}
 		if file, ok := web3ext.Modules[api]; ok {
+			// Load our extension for the module.
 			if err = c.jsre.Compile(fmt.Sprintf("%s.js", api), file); err != nil {
 				return fmt.Errorf("%s.js: %v", api, err)
 			}
 			flatten += fmt.Sprintf("var %s = web3.%s; ", api, api)
+		} else if obj, err := c.jsre.Run("web3." + api); err == nil && obj.IsObject() {
+			// Enable web3.js built-in extension if available.
+			flatten += fmt.Sprintf("var %s = web3.%s; ", api, api)
 		}
 	}
 	if _, err = c.jsre.Run(flatten); err != nil {

containers/docker/develop-alpine/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.4
+FROM alpine:3.5
 
 RUN \
-  apk add --update go git make gcc musl-dev && \
+  apk add --update go git make gcc musl-dev ca-certificates && \
   git clone --depth 1 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \

containers/docker/develop-ubuntu/Dockerfile

@@ -1,17 +1,15 @@
-FROM ubuntu:wily
-MAINTAINER caktux
+FROM ubuntu:xenial
 
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get update && \
-    apt-get upgrade -q -y && \
-    apt-get dist-upgrade -q -y && \
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 923F6CA9 && \
-    echo "deb http://ppa.launchpad.net/ethereum/ethereum-dev/ubuntu wily main" | tee -a /etc/apt/sources.list.d/ethereum.list && \
-    apt-get update && \
-    apt-get install -q -y geth
+RUN \
+  apt-get update && apt-get upgrade -q -y && \
+  apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \
+  git clone --depth 1 https://github.com/ethereum/go-ethereum && \
+  (cd go-ethereum && make geth) && \
+  cp go-ethereum/build/bin/geth /geth && \
+  apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \
+  rm -rf /go-ethereum
 
 EXPOSE 8545
 EXPOSE 30303
 
-ENTRYPOINT ["/usr/bin/geth"]
+ENTRYPOINT ["/geth"]

containers/docker/master-alpine/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.4
+FROM alpine:3.5
 
 RUN \
-  apk add --update go git make gcc musl-dev && \
+  apk add --update go git make gcc musl-dev ca-certificates && \
   git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \

containers/docker/master-ubuntu/Dockerfile

@@ -1,17 +1,15 @@
-FROM ubuntu:wily
-MAINTAINER caktux
+FROM ubuntu:xenial
 
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get update && \
-    apt-get upgrade -q -y && \
-    apt-get dist-upgrade -q -y && \
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 923F6CA9 && \
-    echo "deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu wily main" | tee -a /etc/apt/sources.list.d/ethereum.list && \
-    apt-get update && \
-    apt-get install -q -y geth
+RUN \
+  apt-get update && apt-get upgrade -q -y && \
+  apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \
+  git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \
+  (cd go-ethereum && make geth) && \
+  cp go-ethereum/build/bin/geth /geth && \
+  apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \
+  rm -rf /go-ethereum
 
 EXPOSE 8545
 EXPOSE 30303
 
-ENTRYPOINT ["/usr/bin/geth"]
+ENTRYPOINT ["/geth"]

core/bench_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -168,7 +169,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 	// Time the insertion of the new chain.
 	// State and blocks are stored in the same DB.
 	evmux := new(event.TypeMux)
-	chainman, _ := NewBlockChain(db, &params.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux)
+	chainman, _ := NewBlockChain(db, &params.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux, vm.Config{})
 	defer chainman.Stop()
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -278,7 +279,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
-		chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux))
+		chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 		if err != nil {
 			b.Fatalf("error creating chain: %v", err)
 		}

core/block_validator_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
@@ -39,7 +40,7 @@ func proc() (Validator, *BlockChain) {
 	var mux event.TypeMux
 
 	WriteTestNetGenesisBlock(db)
-	blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux)
+	blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux, vm.Config{})
 	if err != nil {
 		fmt.Println(err)
 	}

core/blockchain.go

@@ -107,12 +107,13 @@ type BlockChain struct {
 	pow       pow.PoW
 	processor Processor // block processor interface
 	validator Validator // block and state validator interface
+	vmConfig  vm.Config
 }
 
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialiser the default Ethereum Validator and
 // Processor.
-func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) {
+func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux, vmConfig vm.Config) (*BlockChain, error) {
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
@@ -128,6 +129,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.P
 		blockCache:   blockCache,
 		futureBlocks: futureBlocks,
 		pow:          pow,
+		vmConfig:     vmConfig,
 	}
 	bc.SetValidator(NewBlockValidator(config, bc, pow))
 	bc.SetProcessor(NewStateProcessor(config, bc))
@@ -851,7 +853,7 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err
 	return
 }
 
-// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. It an error is returned
+// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned
 // it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go).
 func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 	// Do a sanity check that the provided chain is actually ordered and linked
@@ -954,7 +956,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			return i, err
 		}
 		// Process block using the parent state as reference point.
-		receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, vm.Config{})
+		receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.vmConfig)
 		if err != nil {
 			self.reportBlock(block, receipts, err)
 			return i, err
@@ -1004,6 +1006,10 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
 				return i, err
 			}
+			// Write hash preimages
+			if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil {
+				return i, err
+			}
 		case SideStatTy:
 			if glog.V(logger.Detail) {
 				glog.Infof("inserted forked block #%d [%x…] (TD=%v) in %9v: %3d txs %d uncles.", block.Number(), block.Hash().Bytes()[0:4], block.Difficulty(), common.PrettyDuration(time.Since(bstart)), len(block.Transactions()), len(block.Uncles()))
@@ -1082,8 +1088,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		newChain    types.Blocks
 		oldChain    types.Blocks
 		commonBlock *types.Block
-		oldStart    = oldBlock
-		newStart    = newBlock
 		deletedTxs  types.Transactions
 		deletedLogs []*types.Log
 		// collectLogs collects the logs that were generated during the
@@ -1124,7 +1128,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		return fmt.Errorf("Invalid new chain")
 	}
 
-	numSplit := newBlock.Number()
 	for {
 		if oldBlock.Hash() == newBlock.Hash() {
 			commonBlock = oldBlock
@@ -1145,9 +1148,19 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		}
 	}
 
-	if glog.V(logger.Debug) {
-		commonHash := commonBlock.Hash()
-		glog.Infof("Chain split detected @ %x. Reorganising chain from #%v %x to %x", commonHash[:4], numSplit, oldStart.Hash().Bytes()[:4], newStart.Hash().Bytes()[:4])
+	if oldLen := len(oldChain); oldLen > 63 || glog.V(logger.Debug) {
+		newLen := len(newChain)
+		newLast := newChain[0]
+		newFirst := newChain[newLen-1]
+		oldLast := oldChain[0]
+		oldFirst := oldChain[oldLen-1]
+		glog.Infof("Chain split detected after #%v [%x…]. Reorganising chain (-%v +%v blocks), rejecting #%v-#%v [%x…/%x…] in favour of #%v-#%v [%x…/%x…]",
+			commonBlock.Number(), commonBlock.Hash().Bytes()[:4],
+			oldLen, newLen,
+			oldFirst.Number(), oldLast.Number(),
+			oldFirst.Hash().Bytes()[:4], oldLast.Hash().Bytes()[:4],
+			newFirst.Number(), newLast.Number(),
+			newFirst.Hash().Bytes()[:4], newLast.Hash().Bytes()[:4])
 	}
 
 	var addedTxs types.Transactions

core/blockchain_test.go

@@ -53,7 +53,7 @@ func thePow() pow.PoW {
 func theBlockChain(db ethdb.Database, t *testing.T) *BlockChain {
 	var eventMux event.TypeMux
 	WriteTestNetGenesisBlock(db)
-	blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux)
+	blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux, vm.Config{})
 	if err != nil {
 		t.Error("failed creating blockchain:", err)
 		t.FailNow()
@@ -614,7 +614,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 		defer func() { delete(BadHashes, headers[3].Hash()) }()
 	}
 	// Create a new chain manager and check it rolled back the state
-	ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux))
+	ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
@@ -735,7 +735,7 @@ func TestFastVsFullChains(t *testing.T) {
 	archiveDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
 
-	archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux))
+	archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
@@ -743,7 +743,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Fast import the chain as a non-archive node to test
 	fastDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
-	fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux))
+	fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 
 	headers := make([]*types.Header, len(blocks))
 	for i, block := range blocks {
@@ -819,7 +819,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	archiveDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
 
-	archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux))
+	archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
@@ -831,7 +831,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a non-archive node and ensure all pointers are updated
 	fastDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
-	fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux))
+	fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 
 	headers := make([]*types.Header, len(blocks))
 	for i, block := range blocks {
@@ -850,7 +850,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a light node and ensure all pointers are updated
 	lightDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds})
-	light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux))
+	light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{})
 
 	if n, err := light.InsertHeaderChain(headers, 1); err != nil {
 		t.Fatalf("failed to insert header %d: %v", n, err)
@@ -916,7 +916,7 @@ func TestChainTxReorgs(t *testing.T) {
 	})
 	// Import the chain. This runs all block validation rules.
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{})
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		t.Fatalf("failed to insert original chain[%d]: %v", i, err)
 	}
@@ -990,7 +990,7 @@ func TestLogReorgs(t *testing.T) {
 	)
 
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{})
 
 	subs := evmux.Subscribe(RemovedLogsEvent{})
 	chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 2, func(i int, gen *BlockGen) {
@@ -1027,7 +1027,7 @@ func TestReorgSideEvent(t *testing.T) {
 	)
 
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{})
 
 	chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 3, func(i int, gen *BlockGen) {})
 	if _, err := blockchain.InsertChain(chain); err != nil {
@@ -1103,7 +1103,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 	)
 
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{})
 
 	chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *BlockGen) {})
 
@@ -1146,7 +1146,7 @@ func TestEIP155Transition(t *testing.T) {
 		mux        event.TypeMux
 	)
 
-	blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux)
+	blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux, vm.Config{})
 	blocks, _ := GenerateChain(config, genesis, db, 4, func(i int, block *BlockGen) {
 		var (
 			tx      *types.Transaction
@@ -1250,7 +1250,7 @@ func TestEIP161AccountRemoval(t *testing.T) {
 		}
 		mux event.TypeMux
 
-		blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux)
+		blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux, vm.Config{})
 	)
 	blocks, _ := GenerateChain(config, genesis, db, 3, func(i int, block *BlockGen) {
 		var (

core/chain_makers.go

@@ -256,7 +256,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) {
 	// Initialize a fresh chain with only a genesis block
 	genesis, _ := WriteTestNetGenesisBlock(db)
 
-	blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux, vm.Config{})
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil

core/chain_makers_test.go

@@ -21,6 +21,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -81,7 +82,7 @@ func ExampleGenerateChain() {
 
 	// Import the chain. This runs all block validation rules.
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux)
+	blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux, vm.Config{})
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err)
 		return

core/dao_test.go

@@ -20,6 +20,7 @@ import (
 	"math/big"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
@@ -39,12 +40,12 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	proDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(proDb)
 	proConf := &params.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true}
-	proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux))
+	proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 	conDb, _ := ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(conDb)
 	conConf := &params.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false}
-	conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux))
+	conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 	if _, err := proBc.InsertChain(prefix); err != nil {
 		t.Fatalf("pro-fork: failed to import chain prefix: %v", err)
@@ -57,7 +58,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a pro-fork block, and try to feed into the no-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		WriteGenesisBlockForTesting(db)
-		bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux))
+		bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 		blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1))
 		for j := 0; j < len(blocks)/2; j++ {
@@ -78,7 +79,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a no-fork block, and try to feed into the pro-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		WriteGenesisBlockForTesting(db)
-		bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux))
+		bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 		blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1))
 		for j := 0; j < len(blocks)/2; j++ {
@@ -100,7 +101,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(db)
-	bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux))
+	bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1))
 	for j := 0; j < len(blocks)/2; j++ {
@@ -116,7 +117,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	WriteGenesisBlockForTesting(db)
-	bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux))
+	bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{})
 
 	blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1))
 	for j := 0; j < len(blocks)/2; j++ {

core/database_util.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -39,12 +40,13 @@ var (
 	headBlockKey  = []byte("LastBlock")
 	headFastKey   = []byte("LastFast")
 
-	headerPrefix        = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
-	tdSuffix            = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td
-	numSuffix           = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash
-	blockHashPrefix     = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian)
-	bodyPrefix          = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body
-	blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts
+	headerPrefix        = []byte("h")   // headerPrefix + num (uint64 big endian) + hash -> header
+	tdSuffix            = []byte("t")   // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td
+	numSuffix           = []byte("n")   // headerPrefix + num (uint64 big endian) + numSuffix -> hash
+	blockHashPrefix     = []byte("H")   // blockHashPrefix + hash -> num (uint64 big endian)
+	bodyPrefix          = []byte("b")   // bodyPrefix + num (uint64 big endian) + hash -> block body
+	blockReceiptsPrefix = []byte("r")   // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts
+	preimagePrefix      = "secure-key-" // preimagePrefix + hash -> preimage
 
 	txMetaSuffix   = []byte{0x01}
 	receiptsPrefix = []byte("receipts-")
@@ -66,6 +68,9 @@ var (
 	ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error
 
 	mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms
+
+	preimageCounter    = metrics.NewCounter("db/preimage/total")
+	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
 )
 
 // encodeBlockNumber encodes a block number as big endian uint64
@@ -595,6 +600,34 @@ func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
 	return types.BytesToBloom(bloomDat)
 }
 
+// PreimageTable returns a Database instance with the key prefix for preimage entries.
+func PreimageTable(db ethdb.Database) ethdb.Database {
+	return ethdb.NewTable(db, preimagePrefix)
+}
+
+// WritePreimages writes the provided set of preimages to the database. `number` is the
+// current block number, and is used for debug messages only.
+func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash][]byte) error {
+	table := PreimageTable(db)
+	batch := table.NewBatch()
+	hitCount := 0
+	for hash, preimage := range preimages {
+		if _, err := table.Get(hash.Bytes()); err != nil {
+			batch.Put(hash.Bytes(), preimage)
+			hitCount += 1
+		}
+	}
+	preimageCounter.Inc(int64(len(preimages)))
+	preimageHitCounter.Inc(int64(hitCount))
+	if hitCount > 0 {
+		if err := batch.Write(); err != nil {
+			return fmt.Errorf("preimage write fail for block %d: %v", number, err)
+		}
+		glog.V(logger.Debug).Infof("%d preimages in block %d, including %d new", len(preimages), number, hitCount)
+	}
+	return nil
+}
+
 // GetBlockChainVersion reads the version number from db.
 func GetBlockChainVersion(db ethdb.Database) int {
 	var vsn uint

core/state/journal.go

@@ -67,6 +67,9 @@ type (
 	addLogChange struct {
 		txhash common.Hash
 	}
+	addPreimageChange struct {
+		hash common.Hash
+	}
 	touchChange struct {
 		account *common.Address
 		prev    bool
@@ -127,3 +130,7 @@ func (ch addLogChange) undo(s *StateDB) {
 		s.logs[ch.txhash] = logs[:len(logs)-1]
 	}
 }
+
+func (ch addPreimageChange) undo(s *StateDB) {
+	delete(s.preimages, ch.hash)
+}

core/state/statedb.go

@@ -75,6 +75,8 @@ type StateDB struct {
 	logs         map[common.Hash][]*types.Log
 	logSize      uint
 
+	preimages map[common.Hash][]byte
+
 	// Journal of state modifications. This is the backbone of
 	// Snapshot and RevertToSnapshot.
 	journal        journal
@@ -99,6 +101,7 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		refund:            new(big.Int),
 		logs:              make(map[common.Hash][]*types.Log),
+		preimages:         make(map[common.Hash][]byte),
 	}, nil
 }
 
@@ -120,6 +123,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		refund:            new(big.Int),
 		logs:              make(map[common.Hash][]*types.Log),
+		preimages:         make(map[common.Hash][]byte),
 	}, nil
 }
 
@@ -141,6 +145,7 @@ func (self *StateDB) Reset(root common.Hash) error {
 	self.txIndex = 0
 	self.logs = make(map[common.Hash][]*types.Log)
 	self.logSize = 0
+	self.preimages = make(map[common.Hash][]byte)
 	self.clearJournalAndRefund()
 
 	return nil
@@ -199,6 +204,21 @@ func (self *StateDB) Logs() []*types.Log {
 	return logs
 }
 
+// AddPreimage records a SHA3 preimage seen by the VM.
+func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) {
+	if _, ok := self.preimages[hash]; !ok {
+		self.journal = append(self.journal, addPreimageChange{hash: hash})
+		pi := make([]byte, len(preimage))
+		copy(pi, preimage)
+		self.preimages[hash] = pi
+	}
+}
+
+// Preimages returns a list of SHA3 preimages that have been submitted.
+func (self *StateDB) Preimages() map[common.Hash][]byte {
+	return self.preimages
+}
+
 func (self *StateDB) AddRefund(gas *big.Int) {
 	self.journal = append(self.journal, refundChange{prev: new(big.Int).Set(self.refund)})
 	self.refund.Add(self.refund, gas)
@@ -477,8 +497,9 @@ func (self *StateDB) Copy() *StateDB {
 		refund:            new(big.Int).Set(self.refund),
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
+		preimages:         make(map[common.Hash][]byte),
 	}
-	// Copy the dirty states and logs
+	// Copy the dirty states, logs, and preimages
 	for addr := range self.stateObjectsDirty {
 		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty)
 		state.stateObjectsDirty[addr] = struct{}{}
@@ -487,6 +508,9 @@ func (self *StateDB) Copy() *StateDB {
 		state.logs[hash] = make([]*types.Log, len(logs))
 		copy(state.logs[hash], logs)
 	}
+	for hash, preimage := range self.preimages {
+		state.preimages[hash] = preimage
+	}
 	return state
 }
 

core/state_processor.go

@@ -99,7 +99,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, gp *GasPool, s
 	context := NewEVMContext(msg, header, bc)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
-	vmenv := vm.NewEVM(context, statedb, config, vm.Config{})
+	vmenv := vm.NewEVM(context, statedb, config, cfg)
 	// Apply the transaction to the current state (included in the env)
 	_, gas, err := ApplyMessage(vmenv, msg, gp)
 	if err != nil {

core/state_transition.go

@@ -233,9 +233,6 @@ func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *b
 	)
 	if contractCreation {
 		ret, _, vmerr = vmenv.Create(sender, self.data, self.gas, self.value)
-		if homestead && err == vm.ErrCodeStoreOutOfGas {
-			self.gas = Big0
-		}
 	} else {
 		// Increment the nonce for the next transaction
 		self.state.SetNonce(sender.Address(), self.state.GetNonce(sender.Address())+1)

core/types/transaction.go

@@ -134,7 +134,7 @@ func (tx *Transaction) ChainId() *big.Int {
 	return deriveChainId(tx.data.V)
 }
 
-// Protected returns whether the transaction is pretected from replay protection
+// Protected returns whether the transaction is protected from replay protection.
 func (tx *Transaction) Protected() bool {
 	return isProtectedV(tx.data.V)
 }
@@ -198,7 +198,8 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 
 	var V byte
 	if isProtectedV((*big.Int)(dec.V)) {
-		V = byte((new(big.Int).Sub((*big.Int)(dec.V), deriveChainId((*big.Int)(dec.V))).Uint64()) - 35)
+		chainId := deriveChainId((*big.Int)(dec.V)).Uint64()
+		V = byte(dec.V.ToInt().Uint64() - 35 - 2*chainId)
 	} else {
 		V = byte(((*big.Int)(dec.V)).Uint64() - 27)
 	}
@@ -310,16 +311,20 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
 }
 
 func (tx *Transaction) String() string {
-	// make a best guess about the signer and use that to derive
-	// the sender.
-	signer := deriveSigner(tx.data.V)
-
 	var from, to string
-	if f, err := Sender(signer, tx); err != nil { // derive but don't cache
-		from = "[invalid sender: invalid sig]"
+	if tx.data.V != nil {
+		// make a best guess about the signer and use that to derive
+		// the sender.
+		signer := deriveSigner(tx.data.V)
+		if f, err := Sender(signer, tx); err != nil { // derive but don't cache
+			from = "[invalid sender: invalid sig]"
+		} else {
+			from = fmt.Sprintf("%x", f[:])
+		}
 	} else {
-		from = fmt.Sprintf("%x", f[:])
+		from = "[invalid sender: nil V field]"
 	}
+
 	if tx.data.Recipient == nil {
 		to = "[contract creation]"
 	} else {
@@ -332,13 +337,13 @@ func (tx *Transaction) String() string {
 	From:     %s
 	To:       %s
 	Nonce:    %v
-	GasPrice: %v
-	GasLimit  %v
-	Value:    %v
+	GasPrice: %#x
+	GasLimit  %#x
+	Value:    %#x
 	Data:     0x%x
-	V:        0x%x
-	R:        0x%x
-	S:        0x%x
+	V:        %#x
+	R:        %#x
+	S:        %#x
 	Hex:      %x
 `,
 		tx.Hash(),

core/types/transaction_signing.go

@@ -160,7 +160,7 @@ func (s EIP155Signer) PublicKey(tx *Transaction) ([]byte, error) {
 // needs to be in the [R || S || V] format where V is 0 or 1.
 func (s EIP155Signer) WithSignature(tx *Transaction, sig []byte) (*Transaction, error) {
 	if len(sig) != 65 {
-		panic(fmt.Sprintf("wrong size for snature: got %d, want 65", len(sig)))
+		panic(fmt.Sprintf("wrong size for signature: got %d, want 65", len(sig)))
 	}
 
 	cpy := &Transaction{data: tx.data}

core/types/transaction_test.go

@@ -19,6 +19,7 @@ package types
 import (
 	"bytes"
 	"crypto/ecdsa"
+	"encoding/json"
 	"math/big"
 	"testing"
 
@@ -29,7 +30,6 @@ import (
 
 // The values in those tests are from the Transaction Tests
 // at github.com/ethereum/tests.
-
 var (
 	emptyTx = NewTransaction(
 		0,
@@ -190,3 +190,45 @@ func TestTransactionPriceNonceSort(t *testing.T) {
 		}
 	}
 }
+
+// TestTransactionJSON tests serializing/de-serializing to/from JSON.
+func TestTransactionJSON(t *testing.T) {
+	key, err := crypto.GenerateKey()
+	if err != nil {
+		t.Fatalf("could not generate key: %v", err)
+	}
+	signer := NewEIP155Signer(common.Big1)
+
+	for i := uint64(0); i < 25; i++ {
+		var tx *Transaction
+		switch i % 2 {
+		case 0:
+			tx = NewTransaction(i, common.Address{1}, common.Big0, common.Big1, common.Big2, []byte("abcdef"))
+		case 1:
+			tx = NewContractCreation(i, common.Big0, common.Big1, common.Big2, []byte("abcdef"))
+		}
+
+		tx, err := SignTx(tx, signer, key)
+		if err != nil {
+			t.Fatalf("could not sign transaction: %v", err)
+		}
+
+		data, err := json.Marshal(tx)
+		if err != nil {
+			t.Errorf("json.Marshal failed: %v", err)
+		}
+
+		var parsedTx *Transaction
+		if err := json.Unmarshal(data, &parsedTx); err != nil {
+			t.Errorf("json.Unmarshal failed: %v", err)
+		}
+
+		// compare nonce, price, gaslimit, recipient, amount, payload, V, R, S
+		if tx.Hash() != parsedTx.Hash() {
+			t.Errorf("parsed tx differs from original tx, want %v, got %v", tx, parsedTx)
+		}
+		if tx.ChainId().Cmp(parsedTx.ChainId()) != 0 {
+			t.Errorf("invalid chain id, want %d, got %d", tx.ChainId(), parsedTx.ChainId())
+		}
+	}
+}

core/vm/instructions.go

@@ -247,7 +247,12 @@ func opMulmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *S
 
 func opSha3(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	offset, size := stack.pop(), stack.pop()
-	hash := crypto.Keccak256(memory.Get(offset.Int64(), size.Int64()))
+	data := memory.Get(offset.Int64(), size.Int64())
+	hash := crypto.Keccak256(data)
+
+	if env.vmConfig.EnablePreimageRecording {
+		env.StateDB.AddPreimage(common.BytesToHash(hash), data)
+	}
 
 	stack.push(common.BytesToBig(hash))
 	return nil, nil

core/vm/interface.go

@@ -60,6 +60,7 @@ type StateDB interface {
 	Snapshot() int
 
 	AddLog(*types.Log)
+	AddPreimage(common.Hash, []byte)
 }
 
 // Account represents a contract or basic ethereum account.

core/vm/noop.go

@@ -67,3 +67,4 @@ func (NoopStateDB) Empty(common.Address) bool                         { return f
 func (NoopStateDB) RevertToSnapshot(int)                              {}
 func (NoopStateDB) Snapshot() int                                     { return 0 }
 func (NoopStateDB) AddLog(*types.Log)                                 {}
+func (NoopStateDB) AddPreimage(common.Hash, []byte)                   {}

core/vm/vm.go

@@ -44,6 +44,8 @@ type Config struct {
 	NoRecursion bool
 	// Disable gas metering
 	DisableGasMetering bool
+	// Enable recording of SHA3/keccak preimages
+	EnablePreimageRecording bool
 	// JumpTable contains the EVM instruction table. This
 	// may me left uninitialised and will be set the default
 	// table.

crypto/secp256k1/secp256.go

@@ -40,8 +40,6 @@ import (
 	"errors"
 	"math/big"
 	"unsafe"
-
-	"github.com/ethereum/go-ethereum/crypto/randentropy"
 )
 
 var (
@@ -89,13 +87,11 @@ func Sign(msg []byte, seckey []byte) ([]byte, error) {
 	}
 
 	var (
-		msgdata       = (*C.uchar)(unsafe.Pointer(&msg[0]))
-		nonce         = randentropy.GetEntropyCSPRNG(32)
-		noncefunc     = &(*C.secp256k1_nonce_function_default)
-		noncefuncData = unsafe.Pointer(&nonce[0])
-		sigstruct     C.secp256k1_ecdsa_recoverable_signature
+		msgdata   = (*C.uchar)(unsafe.Pointer(&msg[0]))
+		noncefunc = C.secp256k1_nonce_function_rfc6979
+		sigstruct C.secp256k1_ecdsa_recoverable_signature
 	)
-	if C.secp256k1_ecdsa_sign_recoverable(context, &sigstruct, msgdata, seckeydata, noncefunc, noncefuncData) == 0 {
+	if C.secp256k1_ecdsa_sign_recoverable(context, &sigstruct, msgdata, seckeydata, noncefunc, nil) == 0 {
 		return nil, ErrSignFailed
 	}
 

crypto/secp256k1/secp256_test.go

@@ -112,6 +112,24 @@ func TestSignAndRecover(t *testing.T) {
 	}
 }
 
+func TestSignDeterministic(t *testing.T) {
+	_, seckey := generateKeyPair()
+	msg := make([]byte, 32)
+	copy(msg, "hi there")
+
+	sig1, err := Sign(msg, seckey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sig2, err := Sign(msg, seckey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(sig1, sig2) {
+		t.Fatal("signatures not equal")
+	}
+}
+
 func TestRandomMessagesWithSameKey(t *testing.T) {
 	pubkey, seckey := generateKeyPair()
 	keys := func() ([]byte, []byte) {

eth/api.go

@@ -560,3 +560,9 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common.
 	}
 	return nil, errors.New("database inconsistency")
 }
+
+// Preimage is a debug API function that returns the preimage for a sha3 hash, if known.
+func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) {
+	db := core.PreimageTable(api.eth.ChainDb())
+	return db.Get(hash.Bytes())
+}

eth/backend.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/eth/gasprice"
@@ -77,7 +78,6 @@ type Config struct {
 	DatabaseCache      int
 	DatabaseHandles    int
 
-	NatSpec   bool
 	DocRoot   string
 	AutoDAG   bool
 	PowFake   bool
@@ -97,8 +97,7 @@ type Config struct {
 	GpobaseStepUp           int
 	GpobaseCorrectionFactor int
 
-	EnableJit bool
-	ForceJit  bool
+	EnablePreimageRecording bool
 
 	TestGenesisBlock *types.Block   // Genesis block to seed the chain database with (testing only!)
 	TestGenesisState ethdb.Database // Genesis state to seed the database with (testing only!)
@@ -106,7 +105,6 @@ type Config struct {
 
 type LesServer interface {
 	Start(srvr *p2p.Server)
-	Synced()
 	Stop()
 	Protocols() []p2p.Protocol
 }
@@ -140,7 +138,6 @@ type Ethereum struct {
 	etherbase    common.Address
 	solcPath     string
 
-	NatSpec       bool
 	netVersionId  int
 	netRPCService *ethapi.PublicNetAPI
 }
@@ -174,7 +171,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		shutdownChan:   make(chan bool),
 		stopDbUpgrade:  stopDbUpgrade,
 		netVersionId:   config.NetworkId,
-		NatSpec:        config.NatSpec,
 		etherbase:      config.Etherbase,
 		MinerThreads:   config.MinerThreads,
 		AutoDAG:        config.AutoDAG,
@@ -218,7 +214,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 
 	glog.V(logger.Info).Infoln("Chain config:", eth.chainConfig)
 
-	eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux())
+	eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux(), vm.Config{EnablePreimageRecording: config.EnablePreimageRecording})
 	if err != nil {
 		if err == core.ErrNoGenesis {
 			return nil, fmt.Errorf(`No chain found. Please initialise a new chain using the "init" subcommand.`)

eth/downloader/queue.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -101,10 +100,9 @@ type queue struct {
 	stateTaskQueue *prque.Prque             // [eth/63] Priority queue of the hashes to fetch the node data for
 	statePendPool  map[string]*fetchRequest // [eth/63] Currently pending node data retrieval operations
 
-	stateDatabase   ethdb.Database   // [eth/63] Trie database to populate during state reassembly
-	stateScheduler  *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator
-	stateProcessors int32            // [eth/63] Number of currently running state processors
-	stateSchedLock  sync.RWMutex     // [eth/63] Lock serialising access to the state scheduler
+	stateDatabase  ethdb.Database   // [eth/63] Trie database to populate during state reassembly
+	stateScheduler *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator
+	stateWriters   int              // [eth/63] Number of running state DB writer goroutines
 
 	resultCache  []*fetchResult // Downloaded but not yet delivered fetch results
 	resultOffset uint64         // Offset of the first cached fetch result in the block chain
@@ -143,9 +141,6 @@ func (q *queue) Reset() {
 	q.lock.Lock()
 	defer q.lock.Unlock()
 
-	q.stateSchedLock.Lock()
-	defer q.stateSchedLock.Unlock()
-
 	q.closed = false
 	q.mode = FullSync
 	q.fastSyncPivot = 0
@@ -209,13 +204,24 @@ func (q *queue) PendingReceipts() int {
 
 // PendingNodeData retrieves the number of node data entries pending for retrieval.
 func (q *queue) PendingNodeData() int {
-	q.stateSchedLock.RLock()
-	defer q.stateSchedLock.RUnlock()
+	q.lock.Lock()
+	defer q.lock.Unlock()
 
+	return q.pendingNodeDataLocked()
+}
+
+// pendingNodeDataLocked retrieves the number of node data entries pending for retrieval.
+// The caller must hold q.lock.
+func (q *queue) pendingNodeDataLocked() int {
+	var n int
 	if q.stateScheduler != nil {
-		return q.stateScheduler.Pending()
+		n = q.stateScheduler.Pending()
 	}
-	return 0
+	// Ensure that PendingNodeData doesn't return 0 until all state is written.
+	if q.stateWriters > 0 {
+		n++
+	}
+	return n
 }
 
 // InFlightHeaders retrieves whether there are header fetch requests currently
@@ -251,7 +257,7 @@ func (q *queue) InFlightNodeData() bool {
 	q.lock.Lock()
 	defer q.lock.Unlock()
 
-	return len(q.statePendPool)+int(atomic.LoadInt32(&q.stateProcessors)) > 0
+	return len(q.statePendPool)+q.stateWriters > 0
 }
 
 // Idle returns if the queue is fully idle or has some data still inside. This
@@ -264,12 +270,9 @@ func (q *queue) Idle() bool {
 	pending := len(q.blockPendPool) + len(q.receiptPendPool) + len(q.statePendPool)
 	cached := len(q.blockDonePool) + len(q.receiptDonePool)
 
-	q.stateSchedLock.RLock()
 	if q.stateScheduler != nil {
 		queued += q.stateScheduler.Pending()
 	}
-	q.stateSchedLock.RUnlock()
-
 	return (queued + pending + cached) == 0
 }
 
@@ -398,9 +401,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
 				req.Hashes = make(map[common.Hash]int) // Make sure executing requests fail, but don't disappear
 			}
 
-			q.stateSchedLock.Lock()
 			q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase)
-			q.stateSchedLock.Unlock()
 		}
 		inserts = append(inserts, header)
 		q.headerHead = hash
@@ -459,7 +460,7 @@ func (q *queue) countProcessableItems() int {
 				// resultCache has space for fsHeaderForceVerify items. Not
 				// doing this could leave us unable to download the required
 				// amount of headers.
-				if i > 0 || len(q.stateTaskPool) > 0 || q.PendingNodeData() > 0 {
+				if i > 0 || len(q.stateTaskPool) > 0 || q.pendingNodeDataLocked() > 0 {
 					return i
 				}
 				for j := 0; j < fsHeaderForceVerify; j++ {
@@ -524,9 +525,6 @@ func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest {
 func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest {
 	// Create a task generator to fetch status-fetch tasks if all schedules ones are done
 	generator := func(max int) {
-		q.stateSchedLock.Lock()
-		defer q.stateSchedLock.Unlock()
-
 		if q.stateScheduler != nil {
 			for _, hash := range q.stateScheduler.Missing(max) {
 				q.stateTaskPool[hash] = q.stateTaskIndex
@@ -1068,7 +1066,7 @@ func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(int, boo
 		}
 	}
 	// Iterate over the downloaded data and verify each of them
-	accepted, errs := 0, make([]error, 0)
+	errs := make([]error, 0)
 	process := []trie.SyncResult{}
 	for _, blob := range data {
 		// Skip any state trie entries that were not requested
@@ -1079,69 +1077,52 @@ func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(int, boo
 		}
 		// Inject the next state trie item into the processing queue
 		process = append(process, trie.SyncResult{Hash: hash, Data: blob})
-		accepted++
-
 		delete(request.Hashes, hash)
 		delete(q.stateTaskPool, hash)
 	}
-	// Start the asynchronous node state data injection
-	atomic.AddInt32(&q.stateProcessors, 1)
-	go func() {
-		defer atomic.AddInt32(&q.stateProcessors, -1)
-		q.deliverNodeData(process, callback)
-	}()
 	// Return all failed or missing fetches to the queue
 	for hash, index := range request.Hashes {
 		q.stateTaskQueue.Push(hash, float32(index))
 	}
+	if q.stateScheduler == nil {
+		return 0, errNoFetchesPending
+	}
+
+	// Run valid nodes through the trie download scheduler. It writes completed nodes to a
+	// batch, which is committed asynchronously. This may lead to over-fetches because the
+	// scheduler treats everything as written after Process has returned, but it's
+	// unlikely to be an issue in practice.
+	batch := q.stateDatabase.NewBatch()
+	progressed, nproc, procerr := q.stateScheduler.Process(process, batch)
+	q.stateWriters += 1
+	go func() {
+		if procerr == nil {
+			nproc = len(process)
+			procerr = batch.Write()
+		}
+		// Return processing errors through the callback so the sync gets canceled. The
+		// number of writers is decremented prior to the call so PendingNodeData will
+		// return zero when the callback runs.
+		q.lock.Lock()
+		q.stateWriters -= 1
+		q.lock.Unlock()
+		callback(nproc, progressed, procerr)
+		// Wake up WaitResults after the state has been written because it might be
+		// waiting for completion of the pivot block's state download.
+		q.active.Signal()
+	}()
+
 	// If none of the data items were good, it's a stale delivery
 	switch {
 	case len(errs) == 0:
-		return accepted, nil
+		return len(process), nil
 	case len(errs) == len(request.Hashes):
-		return accepted, errStaleDelivery
+		return len(process), errStaleDelivery
 	default:
-		return accepted, fmt.Errorf("multiple failures: %v", errs)
+		return len(process), fmt.Errorf("multiple failures: %v", errs)
 	}
 }
 
-// deliverNodeData is the asynchronous node data processor that injects a batch
-// of sync results into the state scheduler.
-func (q *queue) deliverNodeData(results []trie.SyncResult, callback func(int, bool, error)) {
-	// Wake up WaitResults after the state has been written because it
-	// might be waiting for the pivot block state to get completed.
-	defer q.active.Signal()
-
-	// Process results one by one to permit task fetches in between
-	progressed := false
-	for i, result := range results {
-		q.stateSchedLock.Lock()
-
-		if q.stateScheduler == nil {
-			// Syncing aborted since this async delivery started, bail out
-			q.stateSchedLock.Unlock()
-			callback(i, progressed, errNoFetchesPending)
-			return
-		}
-
-		batch := q.stateDatabase.NewBatch()
-		prog, _, err := q.stateScheduler.Process([]trie.SyncResult{result}, batch)
-		if err != nil {
-			q.stateSchedLock.Unlock()
-			callback(i, progressed, err)
-		}
-		if err = batch.Write(); err != nil {
-			q.stateSchedLock.Unlock()
-			callback(i, progressed, err)
-		}
-
-		// Item processing succeeded, release the lock (temporarily)
-		progressed = progressed || prog
-		q.stateSchedLock.Unlock()
-	}
-	callback(len(results), progressed, nil)
-}
-
 // Prepare configures the result cache to allow accepting and caching inbound
 // fetch results.
 func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64, head *types.Header) {

eth/handler.go

@@ -173,7 +173,7 @@ func NewProtocolManager(config *params.ChainConfig, fastSync bool, networkId int
 		return blockchain.CurrentBlock().NumberU64()
 	}
 	inserter := func(blocks types.Blocks) (int, error) {
-		manager.setSynced() // Mark initial sync done on any fetcher import
+		atomic.StoreUint32(&manager.synced, 1) // Mark initial sync done on any fetcher import
 		return manager.insertChain(blocks)
 	}
 	manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer)

eth/handler_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -469,7 +470,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool
 		db, _         = ethdb.NewMemDatabase()
 		genesis       = core.WriteGenesisBlockForTesting(db)
 		config        = &params.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
-		blockchain, _ = core.NewBlockChain(db, config, pow, evmux)
+		blockchain, _ = core.NewBlockChain(db, config, pow, evmux, vm.Config{})
 	)
 	pm, err := NewProtocolManager(config, false, NetworkId, 1000, evmux, new(testTxPool), pow, blockchain, db)
 	if err != nil {

eth/helper_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -56,7 +57,7 @@ func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core
 		db, _         = ethdb.NewMemDatabase()
 		genesis       = core.WriteGenesisBlockForTesting(db, testBank)
 		chainConfig   = &params.ChainConfig{HomesteadBlock: big.NewInt(0)} // homestead set to 0 because of chain maker
-		blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux)
+		blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{})
 	)
 	chain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator)
 	if _, err := blockchain.InsertChain(chain); err != nil {

eth/sync.go

@@ -181,7 +181,7 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
 	if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil {
 		return
 	}
-	pm.setSynced() // Mark initial sync done
+	atomic.StoreUint32(&pm.synced, 1) // Mark initial sync done
 
 	// If fast sync was enabled, and we synced up, disable it
 	if atomic.LoadUint32(&pm.fastSync) == 1 {
@@ -192,10 +192,3 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
 		}
 	}
 }
-
-// setSynced sets the synced flag and notifies the light server if present
-func (pm *ProtocolManager) setSynced() {
-	if atomic.SwapUint32(&pm.synced, 1) == 0 && pm.lesServer != nil {
-		pm.lesServer.Synced()
-	}
-}

internal/ethapi/api.go

@@ -559,8 +559,34 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr r
 
 // EstimateGas returns an estimate of the amount of gas needed to execute the given transaction.
 func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (*hexutil.Big, error) {
-	_, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber)
-	return (*hexutil.Big)(gas), err
+	// Binary search the gas requirement, as it may be higher than the amount used
+	var lo, hi uint64
+	if (*big.Int)(&args.Gas).BitLen() > 0 {
+		hi = (*big.Int)(&args.Gas).Uint64()
+	} else {
+		// Retrieve the current pending block to act as the gas ceiling
+		block, err := s.b.BlockByNumber(ctx, rpc.PendingBlockNumber)
+		if err != nil {
+			return nil, err
+		}
+		hi = block.GasLimit().Uint64()
+	}
+	for lo+1 < hi {
+		// Take a guess at the gas, and check transaction validity
+		mid := (hi + lo) / 2
+		(*big.Int)(&args.Gas).SetUint64(mid)
+
+		_, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber)
+
+		// If the transaction became invalid or used all the gas (failed), raise the gas limit
+		if err != nil || gas.Cmp((*big.Int)(&args.Gas)) == 0 {
+			lo = mid
+			continue
+		}
+		// Otherwise assume the transaction succeeded, lower the gas limit
+		hi = mid
+	}
+	return (*hexutil.Big)(new(big.Int).SetUint64(hi)), nil
 }
 
 // ExecutionResult groups all structured logs emitted by the EVM

internal/guide/guide.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package guide is a small test suite to ensure snippets in the dev guide work.
+package guide

internal/guide/guide_test.go

@@ -0,0 +1,100 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the code snippets from the developer's guide embedded into
+// Go tests. This ensures that any code published in out guides will not break
+// accidentally via some code update. If some API changes nonetheless that needs
+// modifying this file, please port any modification over into the developer's
+// guide wiki pages too!
+
+package guide
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// Tests that the account management snippets work correctly.
+func TestAccountManagement(t *testing.T) {
+	// Create a temporary folder to work with
+	workdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("Failed to create temporary work dir: %v", err)
+	}
+	defer os.RemoveAll(workdir)
+
+	// Create an encrypted keystore manager with standard crypto parameters
+	am := accounts.NewManager(filepath.Join(workdir, "keystore"), accounts.StandardScryptN, accounts.StandardScryptP)
+
+	// Create a new account with the specified encryption passphrase
+	newAcc, err := am.NewAccount("Creation password")
+	if err != nil {
+		t.Fatalf("Failed to create new account: %v", err)
+	}
+	// Export the newly created account with a different passphrase. The returned
+	// data from this method invocation is a JSON encoded, encrypted key-file
+	jsonAcc, err := am.Export(newAcc, "Creation password", "Export password")
+	if err != nil {
+		t.Fatalf("Failed to export account: %v", err)
+	}
+	// Update the passphrase on the account created above inside the local keystore
+	if err := am.Update(newAcc, "Creation password", "Update password"); err != nil {
+		t.Fatalf("Failed to update account: %v", err)
+	}
+	// Delete the account updated above from the local keystore
+	if err := am.Delete(newAcc, "Update password"); err != nil {
+		t.Fatalf("Failed to delete account: %v", err)
+	}
+	// Import back the account we've exported (and then deleted) above with yet
+	// again a fresh passphrase
+	if _, err := am.Import(jsonAcc, "Export password", "Import password"); err != nil {
+		t.Fatalf("Failed to import account: %v", err)
+	}
+	// Create a new account to sign transactions with
+	signer, err := am.NewAccount("Signer password")
+	if err != nil {
+		t.Fatalf("Failed to create signer account: %v", err)
+	}
+	txHash := common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")
+
+	// Sign a transaction with a single authorization
+	if _, err := am.SignWithPassphrase(signer, "Signer password", txHash.Bytes()); err != nil {
+		t.Fatalf("Failed to sign with passphrase: %v", err)
+	}
+	// Sign a transaction with multiple manually cancelled authorizations
+	if err := am.Unlock(signer, "Signer password"); err != nil {
+		t.Fatalf("Failed to unlock account: %v", err)
+	}
+	if _, err := am.Sign(signer.Address, txHash.Bytes()); err != nil {
+		t.Fatalf("Failed to sign with unlocked account: %v", err)
+	}
+	if err := am.Lock(signer.Address); err != nil {
+		t.Fatalf("Failed to lock account: %v", err)
+	}
+	// Sign a transaction with multiple automatically cancelled authorizations
+	if err := am.TimedUnlock(signer, "Signer password", time.Second); err != nil {
+		t.Fatalf("Failed to time unlock account: %v", err)
+	}
+	if _, err := am.Sign(signer.Address, txHash.Bytes()); err != nil {
+		t.Fatalf("Failed to sign with time unlocked account: %v", err)
+	}
+}

internal/web3ext/web3ext.go

@@ -19,10 +19,8 @@ package web3ext
 
 var Modules = map[string]string{
 	"admin":      Admin_JS,
-	"bzz":        Bzz_JS,
 	"chequebook": Chequebook_JS,
 	"debug":      Debug_JS,
-	"ens":        ENS_JS,
 	"eth":        Eth_JS,
 	"miner":      Miner_JS,
 	"net":        Net_JS,
@@ -32,101 +30,6 @@ var Modules = map[string]string{
 	"txpool":     TxPool_JS,
 }
 
-const Bzz_JS = `
-web3._extend({
-	property: 'bzz',
-	methods:
-	[
-		new web3._extend.Method({
-			name: 'syncEnabled',
-			call: 'bzz_syncEnabled',
-			params: 1,
-			inputFormatter: [null]
-		}),
-		new web3._extend.Method({
-			name: 'swapEnabled',
-			call: 'bzz_swapEnabled',
-			params: 1,
-			inputFormatter: [null]
-		}),
-		new web3._extend.Method({
-			name: 'download',
-			call: 'bzz_download',
-			params: 2,
-			inputFormatter: [null, null]
-		}),
-		new web3._extend.Method({
-			name: 'upload',
-			call: 'bzz_upload',
-			params: 2,
-			inputFormatter: [null, null]
-		}),
-		new web3._extend.Method({
-			name: 'resolve',
-			call: 'bzz_resolve',
-			params: 1,
-			inputFormatter: [null]
-		}),
-		new web3._extend.Method({
-			name: 'get',
-			call: 'bzz_get',
-			params: 1,
-			inputFormatter: [null]
-		}),
-		new web3._extend.Method({
-			name: 'put',
-			call: 'bzz_put',
-			params: 2,
-			inputFormatter: [null, null]
-		}),
-		new web3._extend.Method({
-			name: 'modify',
-			call: 'bzz_modify',
-			params: 4,
-			inputFormatter: [null, null, null, null]
-		})
-	],
-	properties:
-	[
-		new web3._extend.Property({
-			name: 'hive',
-			getter: 'bzz_hive'
-		}),
-		new web3._extend.Property({
-			name: 'info',
-			getter: 'bzz_info',
-		}),
-	]
-});
-`
-
-const ENS_JS = `
-web3._extend({
-  property: 'ens',
-  methods:
-  [
-    new web3._extend.Method({
-			name: 'register',
-			call: 'ens_register',
-			params: 1,
-			inputFormatter: [null]
-		}),
-	new web3._extend.Method({
-			name: 'setContentHash',
-			call: 'ens_setContentHash',
-			params: 2,
-			inputFormatter: [null, null]
-		}),
-	new web3._extend.Method({
-			name: 'resolve',
-			call: 'ens_resolve',
-			params: 1,
-			inputFormatter: [null]
-		}),
-	]
-})
-`
-
 const Chequebook_JS = `
 web3._extend({
   property: 'chequebook',
@@ -385,6 +288,12 @@ web3._extend({
 			call: 'debug_traceTransaction',
 			params: 2,
 			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'preimage',
+			call: 'debug_preimage',
+			params: 1,
+			inputFormatter: [null]
 		})
 	],
 	properties: []
@@ -408,12 +317,6 @@ web3._extend({
 			params: 3,
 			inputFormatter: [web3._extend.formatters.inputTransactionFormatter, web3._extend.utils.fromDecimal, web3._extend.utils.fromDecimal]
 		}),
-		new web3._extend.Method({
-			name: 'getNatSpec',
-			call: 'eth_getNatSpec',
-			params: 1,
-			inputFormatter: [web3._extend.formatters.inputTransactionFormatter]
-		}),
 		new web3._extend.Method({
 			name: 'signTransaction',
 			call: 'eth_signTransaction',

les/backend.go

@@ -66,7 +66,6 @@ type LightEthereum struct {
 	solcPath       string
 	solc           *compiler.Solidity
 
-	NatSpec       bool
 	netVersionId  int
 	netRPCService *ethapi.PublicNetAPI
 }
@@ -95,7 +94,6 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) {
 		pow:            pow,
 		shutdownChan:   make(chan bool),
 		netVersionId:   config.NetworkId,
-		NatSpec:        config.NatSpec,
 		solcPath:       config.SolcPath,
 	}
 

les/handler.go

@@ -160,9 +160,6 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network
 				if manager.serverPool != nil {
 					addr := p.RemoteAddr().(*net.TCPAddr)
 					entry = manager.serverPool.connect(peer, addr.IP, uint16(addr.Port))
-					if entry == nil {
-						return fmt.Errorf("unwanted connection")
-					}
 				}
 				peer.poolEntry = entry
 				select {

les/helper_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -143,7 +144,7 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor
 		odr = NewLesOdr(db)
 		chain, _ = light.NewLightChain(odr, chainConfig, pow, evmux)
 	} else {
-		blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux)
+		blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{})
 		gchain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator)
 		if _, err := blockchain.InsertChain(gchain); err != nil {
 			panic(err)

les/server.go

@@ -42,9 +42,7 @@ type LesServer struct {
 	fcManager       *flowcontrol.ClientManager // nil if our node is client only
 	fcCostStats     *requestCostStats
 	defParams       *flowcontrol.ServerParams
-	srvr            *p2p.Server
-	synced, stopped bool
-	lock            sync.Mutex
+	stopped         bool
 }
 
 func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) {
@@ -70,35 +68,13 @@ func (s *LesServer) Protocols() []p2p.Protocol {
 	return s.protocolManager.SubProtocols
 }
 
-// Start only starts the actual service if the ETH protocol has already been synced,
-// otherwise it will be started by Synced()
+// Start starts the LES server
 func (s *LesServer) Start(srvr *p2p.Server) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	s.srvr = srvr
-	if s.synced {
-		s.protocolManager.Start(s.srvr)
-	}
-}
-
-// Synced notifies the server that the ETH protocol has been synced and LES service can be started
-func (s *LesServer) Synced() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	s.synced = true
-	if s.srvr != nil && !s.stopped {
-		s.protocolManager.Start(s.srvr)
-	}
+	s.protocolManager.Start(srvr)
 }
 
 // Stop stops the LES service
 func (s *LesServer) Stop() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	s.stopped = true
 	s.fcCostStats.store()
 	s.fcManager.Stop()
 	go func() {

les/serverpool.go

@@ -160,10 +160,10 @@ func (pool *serverPool) connect(p *peer, ip net.IP, port uint16) *poolEntry {
 	defer pool.lock.Unlock()
 	entry := pool.entries[p.ID()]
 	if entry == nil {
-		return nil
+		entry = pool.findOrNewNode(p.ID(), ip, port)
 	}
 	glog.V(logger.Debug).Infof("connecting to %v, state: %v", p.id, entry.state)
-	if entry.state != psDialed {
+	if entry.state == psConnected || entry.state == psRegistered {
 		return nil
 	}
 	pool.connWg.Add(1)
@@ -250,11 +250,17 @@ type poolStatAdjust struct {
 
 // adjustBlockDelay adjusts the block announce delay statistics of a node
 func (pool *serverPool) adjustBlockDelay(entry *poolEntry, time time.Duration) {
+	if entry == nil {
+		return
+	}
 	pool.adjustStats <- poolStatAdjust{pseBlockDelay, entry, time}
 }
 
 // adjustResponseTime adjusts the request response time statistics of a node
 func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, timeout bool) {
+	if entry == nil {
+		return
+	}
 	if timeout {
 		pool.adjustStats <- poolStatAdjust{pseResponseTimeout, entry, time}
 	} else {
@@ -342,7 +348,9 @@ func (pool *serverPool) selectPeerWait(reqID uint64, canSend func(*peer) (bool,
 func (pool *serverPool) eventLoop() {
 	lookupCnt := 0
 	var convTime mclock.AbsTime
-	pool.discSetPeriod <- time.Millisecond * 100
+	if pool.discSetPeriod != nil {
+		pool.discSetPeriod <- time.Millisecond * 100
+	}
 	for {
 		select {
 		case entry := <-pool.timeout:
@@ -375,39 +383,7 @@ func (pool *serverPool) eventLoop() {
 
 		case node := <-pool.discNodes:
 			pool.lock.Lock()
-			now := mclock.Now()
-			id := discover.NodeID(node.ID)
-			entry := pool.entries[id]
-			if entry == nil {
-				glog.V(logger.Debug).Infof("discovered %v", node.String())
-				entry = &poolEntry{
-					id:         id,
-					addr:       make(map[string]*poolEntryAddress),
-					addrSelect: *newWeightedRandomSelect(),
-					shortRetry: shortRetryCnt,
-				}
-				pool.entries[id] = entry
-				// initialize previously unknown peers with good statistics to give a chance to prove themselves
-				entry.connectStats.add(1, initStatsWeight)
-				entry.delayStats.add(0, initStatsWeight)
-				entry.responseStats.add(0, initStatsWeight)
-				entry.timeoutStats.add(0, initStatsWeight)
-			}
-			entry.lastDiscovered = now
-			addr := &poolEntryAddress{
-				ip:   node.IP,
-				port: node.TCP,
-			}
-			if a, ok := entry.addr[addr.strKey()]; ok {
-				addr = a
-			} else {
-				entry.addr[addr.strKey()] = addr
-			}
-			addr.lastSeen = now
-			entry.addrSelect.update(addr)
-			if !entry.known {
-				pool.newQueue.setLatest(entry)
-			}
+			entry := pool.findOrNewNode(discover.NodeID(node.ID), node.IP, node.TCP)
 			pool.updateCheckDial(entry)
 			pool.lock.Unlock()
 
@@ -419,12 +395,16 @@ func (pool *serverPool) eventLoop() {
 				lookupCnt++
 				if pool.fastDiscover && (lookupCnt == 50 || time.Duration(mclock.Now()-convTime) > time.Minute) {
 					pool.fastDiscover = false
-					pool.discSetPeriod <- time.Minute
+					if pool.discSetPeriod != nil {
+						pool.discSetPeriod <- time.Minute
+					}
 				}
 			}
 
 		case <-pool.quit:
-			close(pool.discSetPeriod)
+			if pool.discSetPeriod != nil {
+				close(pool.discSetPeriod)
+			}
 			pool.connWg.Wait()
 			pool.saveNodes()
 			pool.wg.Done()
@@ -434,6 +414,42 @@ func (pool *serverPool) eventLoop() {
 	}
 }
 
+func (pool *serverPool) findOrNewNode(id discover.NodeID, ip net.IP, port uint16) *poolEntry {
+	now := mclock.Now()
+	entry := pool.entries[id]
+	if entry == nil {
+		glog.V(logger.Debug).Infof("discovered %v", id.String())
+		entry = &poolEntry{
+			id:         id,
+			addr:       make(map[string]*poolEntryAddress),
+			addrSelect: *newWeightedRandomSelect(),
+			shortRetry: shortRetryCnt,
+		}
+		pool.entries[id] = entry
+		// initialize previously unknown peers with good statistics to give a chance to prove themselves
+		entry.connectStats.add(1, initStatsWeight)
+		entry.delayStats.add(0, initStatsWeight)
+		entry.responseStats.add(0, initStatsWeight)
+		entry.timeoutStats.add(0, initStatsWeight)
+	}
+	entry.lastDiscovered = now
+	addr := &poolEntryAddress{
+		ip:   ip,
+		port: port,
+	}
+	if a, ok := entry.addr[addr.strKey()]; ok {
+		addr = a
+	} else {
+		entry.addr[addr.strKey()] = addr
+	}
+	addr.lastSeen = now
+	entry.addrSelect.update(addr)
+	if !entry.known {
+		pool.newQueue.setLatest(entry)
+	}
+	return entry
+}
+
 // loadNodes loads known nodes and their statistics from the database
 func (pool *serverPool) loadNodes() {
 	enc, err := pool.db.Get(pool.dbKey)

light/odr_test.go

@@ -251,7 +251,7 @@ func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) {
 	)
 	core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds})
 	// Assemble the test environment
-	blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux)
+	blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{})
 	chainConfig := &params.ChainConfig{HomesteadBlock: new(big.Int)}
 	gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, 4, testChainGen)
 	if _, err := blockchain.InsertChain(gchain); err != nil {

light/txpool_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
@@ -88,7 +89,7 @@ func TestTxPool(t *testing.T) {
 	)
 	core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds})
 	// Assemble the test environment
-	blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux)
+	blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{})
 	chainConfig := &params.ChainConfig{HomesteadBlock: new(big.Int)}
 	gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, poolTestBlocks, txPoolTestChainGen)
 	if _, err := blockchain.InsertChain(gchain); err != nil {

light/vm_env.go

@@ -45,6 +45,8 @@ func (s *VMState) Error() error {
 
 func (s *VMState) AddLog(log *types.Log) {}
 
+func (s *VMState) AddPreimage(hash common.Hash, preimage []byte) {}
+
 // errHandler handles and stores any state error that happens during execution.
 func (s *VMState) errHandler(err error) {
 	if err != nil && s.err == nil {

mobile/accounts.go

@@ -103,7 +103,7 @@ func (am *AccountManager) GetAccounts() *Accounts {
 // DeleteAccount deletes the key matched by account if the passphrase is correct.
 // If a contains no filename, the address must match a unique key.
 func (am *AccountManager) DeleteAccount(account *Account, passphrase string) error {
-	return am.manager.DeleteAccount(accounts.Account{
+	return am.manager.Delete(accounts.Account{
 		Address: account.account.Address,
 		File:    account.account.File,
 	}, passphrase)

params/version.go

@@ -21,7 +21,7 @@ import "fmt"
 const (
 	VersionMajor = 1        // Major version component of the current release
 	VersionMinor = 5        // Minor version component of the current release
-	VersionPatch = 7        // Patch version component of the current release
+	VersionPatch = 8        // Patch version component of the current release
 	VersionMeta  = "stable" // Version metadata to append to the version string
 )
 

swarm/api/api.go

@@ -19,6 +19,7 @@ package api
 import (
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strings"
 	"sync"
@@ -71,6 +72,7 @@ type ErrResolve error
 
 // DNS Resolver
 func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
+	glog.V(logger.Detail).Infof("Resolving : %v", hostPort)
 	if hashMatcher.MatchString(hostPort) || self.dns == nil {
 		glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort)
 		return storage.Key(common.Hex2Bytes(hostPort)), nil
@@ -86,8 +88,10 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error
 	glog.V(logger.Detail).Infof("host lookup: %v -> %v", err)
 	return contentHash[:], err
 }
-
-func parse(uri string) (hostPort, path string) {
+func Parse(uri string) (hostPort, path string) {
+	if uri == "" {
+		return
+	}
 	parts := slashes.Split(uri, 3)
 	var i int
 	if len(parts) == 0 {
@@ -111,7 +115,7 @@ func parse(uri string) (hostPort, path string) {
 }
 
 func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
-	hostPort, path = parse(uri)
+	hostPort, path = Parse(uri)
 	//resolving host and port
 	contentHash, err := self.Resolve(hostPort, nameresolver)
 	glog.V(logger.Debug).Infof("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)
@@ -153,7 +157,9 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 	}
 
 	glog.V(logger.Detail).Infof("getEntry(%s)", path)
+
 	entry, _ := trie.getEntry(path)
+
 	if entry != nil {
 		key = common.Hex2Bytes(entry.Hash)
 		status = entry.Status
@@ -161,6 +167,7 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 		glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType)
 		reader = self.dpa.Retrieve(key)
 	} else {
+		status = http.StatusNotFound
 		err = fmt.Errorf("manifest entry for '%s' not found", path)
 		glog.V(logger.Warn).Infof("%v", err)
 	}

swarm/api/config.go

@@ -85,10 +85,17 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 		NetworkId:     networkId,
 	}
 	data, err = ioutil.ReadFile(confpath)
+
+	// if not set in function param, then set default for swarm network, will be overwritten by config file if present
+	if networkId == 0 {
+		self.NetworkId = network.NetworkId
+	}
+
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return
 		}
+
 		// file does not exist
 		// write out config file
 		err = self.Save()
@@ -97,6 +104,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 		}
 		return
 	}
+
 	// file exists, deserialise
 	err = json.Unmarshal(data, self)
 	if err != nil {
@@ -109,6 +117,12 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 	if keyhex != self.BzzKey {
 		return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey)
 	}
+
+	// if set in function param, replace id set from config file
+	if networkId != 0 {
+		self.NetworkId = networkId
+	}
+
 	self.Swap.SetKey(prvKey)
 
 	if (self.EnsRoot == common.Address{}) {

swarm/api/http/roundtripper.go

@@ -44,9 +44,7 @@ If Host is left empty, localhost is assumed.
 
 Using a public gateway, the above few lines gives you the leanest
 bzz-scheme aware read-only http client. You really only ever need this
-if you need go-native swarm access to bzz addresses, e.g.,
-github.com/ethereum/go-ethereum/common/natspec
-
+if you need go-native swarm access to bzz addresses.
 */
 
 type RoundTripper struct {

swarm/api/http/server.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/rs/cors"
 )
 
@@ -194,17 +195,34 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		}
 	case r.Method == "GET" || r.Method == "HEAD":
 		path = trailingSlashes.ReplaceAllString(path, "")
+		if path == "" {
+			http.Error(w, "Empty path not allowed", http.StatusBadRequest)
+			return
+		}
 		if raw {
-			// resolving host
-			key, err := a.Resolve(path, nameresolver)
-			if err != nil {
-				glog.V(logger.Error).Infof("%v", err)
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
+			var reader storage.LazySectionReader
+			parsedurl, _ := api.Parse(path)
+
+			if parsedurl == path {
+				key, err := a.Resolve(parsedurl, nameresolver)
+				if err != nil {
+					glog.V(logger.Error).Infof("%v", err)
+					http.Error(w, err.Error(), http.StatusBadRequest)
+					return
+				}
+				reader = a.Retrieve(key)
+			} else {
+				var status int
+				readertmp, _, status, err := a.Get(path, nameresolver)
+				if err != nil {
+					http.Error(w, err.Error(), status)
+					return
+				}
+				reader = readertmp
 			}
 
 			// retrieving content
-			reader := a.Retrieve(key)
+
 			quitC := make(chan bool)
 			size, err := reader.Size(quitC)
 			if err != nil {

swarm/api/http/server_test.go

@@ -0,0 +1,133 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func TestBzzrGetPath(t *testing.T) {
+
+	var err error
+
+	maxproxyattempts := 3
+
+	testmanifest := []string{
+		`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
+		`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
+		`{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`,
+	}
+
+	testrequests := make(map[string]int)
+	testrequests["/"] = 0
+	testrequests["/a"] = 1
+	testrequests["/a/b"] = 2
+	testrequests["/x"] = 0
+	testrequests[""] = 0
+
+	expectedfailrequests := []string{"", "/x"}
+
+	reader := [3]*bytes.Reader{}
+
+	key := [3]storage.Key{}
+
+	dir, _ := ioutil.TempDir("", "bzz-storage-test")
+
+	storeparams := &storage.StoreParams{
+		ChunkDbPath:   dir,
+		DbCapacity:    5000000,
+		CacheCapacity: 5000,
+		Radius:        0,
+	}
+
+	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
+	dpa := &storage.DPA{
+		Chunker:    chunker,
+		ChunkStore: localStore,
+	}
+	dpa.Start()
+	defer dpa.Stop()
+
+	wg := &sync.WaitGroup{}
+
+	for i, mf := range testmanifest {
+		reader[i] = bytes.NewReader([]byte(mf))
+		key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		wg.Wait()
+	}
+
+	a := api.NewApi(dpa, nil)
+
+	/// \todo iterate port numbers up if fail
+	StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
+	// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
+	// if we fix it we don't need maxproxyattempts anymore either
+	time.Sleep(1000 * time.Millisecond)
+	for i := 0; i <= maxproxyattempts; i++ {
+		_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+		if i == maxproxyattempts {
+			t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
+		} else if err != nil {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		break
+	}
+
+	for k, v := range testrequests {
+		var resp *http.Response
+		var respbody []byte
+
+		url := "http://127.0.0.1:8504/bzzr:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
+		}
+		resp, err = http.Get(url)
+		defer resp.Body.Close()
+		respbody, err = ioutil.ReadAll(resp.Body)
+
+		if string(respbody) != testmanifest[v] {
+			isexpectedfailrequest := false
+
+			for _, r := range expectedfailrequests {
+				if k[:] == r {
+					isexpectedfailrequest = true
+				}
+			}
+			if isexpectedfailrequest == false {
+				t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(respbody))
+			}
+		}
+	}
+
+}

swarm/api/manifest.go

@@ -302,7 +302,8 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 	if (len(path) >= epl) && (path[:epl] == entry.Path) {
 		glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType)
 		if entry.ContentType == manifestType {
-			if self.loadSubTrie(entry, quitC) != nil {
+			err := self.loadSubTrie(entry, quitC)
+			if err != nil {
 				return nil, 0
 			}
 			entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
@@ -312,8 +313,6 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 		} else {
 			pos = epl
 		}
-	} else {
-		entry = nil
 	}
 	return
 }

tests/block_test_util.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -172,7 +173,7 @@ func runBlockTest(homesteadBlock, daoForkBlock, gasPriceFork *big.Int, test *Blo
 	core.WriteHeadBlockHash(db, test.Genesis.Hash())
 	evmux := new(event.TypeMux)
 	config := &params.ChainConfig{HomesteadBlock: homesteadBlock, DAOForkBlock: daoForkBlock, DAOForkSupport: true, EIP150Block: gasPriceFork}
-	chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux)
+	chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux, vm.Config{})
 	if err != nil {
 		return err
 	}

whisper/mailserver/mailserver.go

@@ -0,0 +1,170 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package mailserver
+
+import (
+	"bytes"
+	"encoding/binary"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/rlp"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const MailServerKeyName = "958e04ab302fb36ad2616a352cbac79d"
+
+type WMailServer struct {
+	db  *leveldb.DB
+	w   *whisper.Whisper
+	pow float64
+	key []byte
+}
+
+type DBKey struct {
+	timestamp uint32
+	hash      common.Hash
+	raw       []byte
+}
+
+func NewDbKey(t uint32, h common.Hash) *DBKey {
+	const sz = common.HashLength + 4
+	var k DBKey
+	k.timestamp = t
+	k.hash = h
+	k.raw = make([]byte, sz)
+	binary.BigEndian.PutUint32(k.raw, k.timestamp)
+	copy(k.raw[4:], k.hash[:])
+	return &k
+}
+
+func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) {
+	var err error
+	if len(path) == 0 {
+		utils.Fatalf("DB file is not specified")
+	}
+
+	if len(password) == 0 {
+		utils.Fatalf("Password is not specified for MailServer")
+	}
+
+	s.db, err = leveldb.OpenFile(path, nil)
+	if err != nil {
+		utils.Fatalf("Failed to open DB file: %s", err)
+	}
+
+	s.w = shh
+	s.pow = pow
+
+	err = s.w.AddSymKey(MailServerKeyName, []byte(password))
+	if err != nil {
+		utils.Fatalf("Failed to create symmetric key for MailServer: %s", err)
+	}
+	s.key = s.w.GetSymKey(MailServerKeyName)
+}
+
+func (s *WMailServer) Close() {
+	if s.db != nil {
+		s.db.Close()
+	}
+}
+
+func (s *WMailServer) Archive(env *whisper.Envelope) {
+	key := NewDbKey(env.Expiry-env.TTL, env.Hash())
+	rawEnvelope, err := rlp.EncodeToBytes(env)
+	if err != nil {
+		glog.V(logger.Error).Infof("rlp.EncodeToBytes failed: %s", err)
+	} else {
+		err = s.db.Put(key.raw, rawEnvelope, nil)
+		if err != nil {
+			glog.V(logger.Error).Infof("Writing to DB failed: %s", err)
+		}
+	}
+}
+
+func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) {
+	ok, lower, upper, topic := s.validate(peer, request)
+	if !ok {
+		return
+	}
+
+	var err error
+	var zero common.Hash
+	var empty whisper.TopicType
+	kl := NewDbKey(lower, zero)
+	ku := NewDbKey(upper, zero)
+	i := s.db.NewIterator(&util.Range{Start: kl.raw, Limit: ku.raw}, nil)
+	defer i.Release()
+
+	for i.Next() {
+		var envelope whisper.Envelope
+		err = rlp.DecodeBytes(i.Value(), &envelope)
+		if err != nil {
+			glog.V(logger.Error).Infof("RLP decoding failed: %s", err)
+		}
+
+		if topic == empty || envelope.Topic == topic {
+			err = s.w.SendP2PDirect(peer, &envelope)
+			if err != nil {
+				glog.V(logger.Error).Infof("Failed to send direct message to peer: %s", err)
+				return
+			}
+		}
+	}
+
+	err = i.Error()
+	if err != nil {
+		glog.V(logger.Error).Infof("Level DB iterator error: %s", err)
+	}
+}
+
+func (s *WMailServer) validate(peer *whisper.Peer, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) {
+	var topic whisper.TopicType
+	if s.pow > 0.0 && request.PoW() < s.pow {
+		return false, 0, 0, topic
+	}
+
+	f := whisper.Filter{KeySym: s.key}
+	decrypted := request.Open(&f)
+	if decrypted == nil {
+		glog.V(logger.Warn).Infof("Failed to decrypt p2p request")
+		return false, 0, 0, topic
+	}
+
+	if len(decrypted.Payload) < 8 {
+		glog.V(logger.Warn).Infof("Undersized p2p request")
+		return false, 0, 0, topic
+	}
+
+	if bytes.Equal(peer.ID(), decrypted.Signature) {
+		glog.V(logger.Warn).Infof("Wrong signature of p2p request")
+		return false, 0, 0, topic
+	}
+
+	lower := binary.BigEndian.Uint32(decrypted.Payload[:4])
+	upper := binary.BigEndian.Uint32(decrypted.Payload[4:8])
+
+	if len(decrypted.Payload) >= 8+whisper.TopicLength {
+		topic = whisper.BytesToTopic(decrypted.Payload[8:])
+	}
+
+	return true, lower, upper, topic
+}

whisper/shhapi/api.go

@@ -93,12 +93,12 @@ func (api *PublicWhisperAPI) MarkPeerTrusted(peerID hexutil.Bytes) error {
 // data contains parameters (time frame, payment details, etc.), required
 // by the remote email-like server. Whisper is not aware about the data format,
 // it will just forward the raw data to the server.
-func (api *PublicWhisperAPI) RequestHistoricMessages(peerID hexutil.Bytes, data hexutil.Bytes) error {
-	if api.whisper == nil {
-		return whisperOffLineErr
-	}
-	return api.whisper.RequestHistoricMessages(peerID, data)
-}
+//func (api *PublicWhisperAPI) RequestHistoricMessages(peerID hexutil.Bytes, data hexutil.Bytes) error {
+//	if api.whisper == nil {
+//		return whisperOffLineErr
+//	}
+//	return api.whisper.RequestHistoricMessages(peerID, data)
+//}
 
 // HasIdentity checks if the whisper node is configured with the private key
 // of the specified public pair.

whisper/whisperv5/doc.go

@@ -83,5 +83,5 @@ func (e unknownVersionError) Error() string {
 // in order to bypass the expiry checks.
 type MailServer interface {
 	Archive(env *Envelope)
-	DeliverMail(whisperPeer *Peer, data []byte)
+	DeliverMail(whisperPeer *Peer, request *Envelope)
 }

whisper/whisperv5/message_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 func copyFromBuf(dst []byte, src []byte, beg int) int {
@@ -311,3 +312,35 @@ func TestEncryptWithZeroKey(t *testing.T) {
 		t.Fatalf("wrapped with nil key, seed: %d.", seed)
 	}
 }
+
+func TestRlpEncode(t *testing.T) {
+	InitSingleTest()
+
+	params, err := generateMessageParams()
+	if err != nil {
+		t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
+	}
+	msg := NewSentMessage(params)
+	env, err := msg.Wrap(params)
+	if err != nil {
+		t.Fatalf("wrapped with zero key, seed: %d.", seed)
+	}
+
+	raw, err := rlp.EncodeToBytes(env)
+	if err != nil {
+		t.Fatalf("RLP encode failed: %s.", err)
+	}
+
+	var decoded Envelope
+	rlp.DecodeBytes(raw, &decoded)
+	if err != nil {
+		t.Fatalf("RLP decode failed: %s.", err)
+	}
+
+	he := env.Hash()
+	hd := decoded.Hash()
+
+	if he != hd {
+		t.Fatalf("Hashes are not equal: %x vs. %x", he, hd)
+	}
+}

whisper/whisperv5/peer.go

@@ -175,3 +175,8 @@ func (p *Peer) broadcast() error {
 	glog.V(logger.Detail).Infoln(p.peer, "broadcasted", len(transmit), "message(s)")
 	return nil
 }
+
+func (p *Peer) ID() []byte {
+	id := p.peer.ID()
+	return id[:]
+}

whisper/whisperv5/whisper.go

@@ -31,7 +31,6 @@ import (
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/crypto/pbkdf2"
 	set "gopkg.in/fatih/set.v0"
 )
@@ -125,13 +124,13 @@ func (w *Whisper) MarkPeerTrusted(peerID []byte) error {
 	return nil
 }
 
-func (w *Whisper) RequestHistoricMessages(peerID []byte, data []byte) error {
+func (w *Whisper) RequestHistoricMessages(peerID []byte, envelope *Envelope) error {
 	p, err := w.getPeer(peerID)
 	if err != nil {
 		return err
 	}
 	p.trusted = true
-	return p2p.Send(p.ws, p2pRequestCode, data)
+	return p2p.Send(p.ws, p2pRequestCode, envelope)
 }
 
 func (w *Whisper) SendP2PMessage(peerID []byte, envelope *Envelope) error {
@@ -142,6 +141,10 @@ func (w *Whisper) SendP2PMessage(peerID []byte, envelope *Envelope) error {
 	return p2p.Send(p.ws, p2pCode, envelope)
 }
 
+func (w *Whisper) SendP2PDirect(peer *Peer, envelope *Envelope) error {
+	return p2p.Send(peer.ws, p2pCode, envelope)
+}
+
 // NewIdentity generates a new cryptographic identity for the client, and injects
 // it into the known identities for message decryption.
 func (w *Whisper) NewIdentity() *ecdsa.PrivateKey {
@@ -347,9 +350,6 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
 					return fmt.Errorf("invalid envelope")
 				}
 				p.mark(envelope)
-				if wh.mailServer != nil {
-					wh.mailServer.Archive(envelope)
-				}
 			}
 		case p2pCode:
 			// peer-to-peer message, sent directly to peer bypassing PoW checks, etc.
@@ -357,25 +357,22 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
 			// therefore might not satisfy the PoW, expiry and other requirements.
 			// these messages are only accepted from the trusted peer.
 			if p.trusted {
-				var envelopes []*Envelope
-				if err := packet.Decode(&envelopes); err != nil {
+				var envelope Envelope
+				if err := packet.Decode(&envelope); err != nil {
 					glog.V(logger.Warn).Infof("%v: failed to decode direct message: [%v], peer will be disconnected", p.peer, err)
 					return fmt.Errorf("garbage received (directMessage)")
 				}
-				for _, envelope := range envelopes {
-					wh.postEvent(envelope, true)
-				}
+				wh.postEvent(&envelope, true)
 			}
 		case p2pRequestCode:
 			// Must be processed if mail server is implemented. Otherwise ignore.
 			if wh.mailServer != nil {
-				s := rlp.NewStream(packet.Payload, uint64(packet.Size))
-				data, err := s.Bytes()
-				if err == nil {
-					wh.mailServer.DeliverMail(p, data)
-				} else {
-					glog.V(logger.Error).Infof("%v: bad requestHistoricMessages received: [%v]", p.peer, err)
+				var request Envelope
+				if err := packet.Decode(&request); err != nil {
+					glog.V(logger.Warn).Infof("%v: failed to decode p2p request message: [%v], peer will be disconnected", p.peer, err)
+					return fmt.Errorf("garbage received (p2p request)")
 				}
+				wh.mailServer.DeliverMail(p, &request)
 			}
 		default:
 			// New message types might be implemented in the future versions of Whisper.
@@ -454,6 +451,9 @@ func (wh *Whisper) add(envelope *Envelope) error {
 	} else {
 		glog.V(logger.Detail).Infof("cached whisper envelope [%x]: %v\n", envelope.Hash(), envelope)
 		wh.postEvent(envelope, false) // notify the local node about the new message
+		if wh.mailServer != nil {
+			wh.mailServer.Archive(envelope)
+		}
 	}
 	return nil
 }

whisper/whisperv5/whisper_test.go

@@ -57,12 +57,6 @@ func TestWhisperBasic(t *testing.T) {
 	if err := w.MarkPeerTrusted(peerID); err == nil {
 		t.Fatalf("failed MarkPeerTrusted.")
 	}
-	if err := w.RequestHistoricMessages(peerID, peerID); err == nil {
-		t.Fatalf("failed RequestHistoricMessages.")
-	}
-	if err := w.SendP2PMessage(peerID, nil); err == nil {
-		t.Fatalf("failed SendP2PMessage.")
-	}
 	exist := w.HasSymKey("non-existing")
 	if exist {
 		t.Fatalf("failed HasSymKey.")