Compare commits

...

36 Commits

Author SHA1 Message Date
Péter Szilágyi
66432f3821 params: release geth 1.8.7 2018-05-02 14:01:38 +03:00
Martin Holst Swende
5d4d79ae26 cmd/clef: documentation about setup (#16568)
clef: documentation about setup
2018-05-02 13:31:05 +03:00
Péter Szilágyi
6a01363d1d Merge pull request #16644 from ligi/reduce_aar_size
build: Add ldflags "-s -w" when building aar
2018-05-02 12:23:24 +03:00
ligi
58c4e033f4 build: Add ldflags -s -w when building aar
Smaller size on mobile is always good.
Might also solve our maven central upload problem
2018-05-02 11:21:24 +02:00
Péter Szilágyi
5449139ca2 Merge pull request #16569 from holiman/evm_blocknum
cmd/evm: use block number from genesis
2018-05-02 11:48:38 +03:00
Péter Szilágyi
579ac6287b Merge pull request #16576 from CrispinFlowerday/bugfix/local_underpriced_txs
core: ensure local transactions aren't discarded as underpriced
2018-05-02 11:34:47 +03:00
kiel barry
a7720b5926 core: golint updates for this or self warning (#16633) 2018-05-02 11:27:59 +03:00
kiel barry
670bae4cd3 internal: golint updates for this or self warning (#16634) 2018-05-02 11:26:21 +03:00
Eli
4a8d5d2b1e trie: golint iterator fixes (#16639) 2018-05-02 11:24:34 +03:00
Eli
d76c5ca532 tests: golint fixes for tests directory (#16640) 2018-05-02 11:20:19 +03:00
kiel barry
c1ea527573 accounts: golint updates for this or self warning (#16627) 2018-05-02 11:17:52 +03:00
Martin Holst Swende
8dfa4f46a9 evm/main: use blocknumber from genesis 2018-05-02 10:17:00 +02:00
Crispin Flowerday
0afd767537 core: ensure local transactions aren't discarded as underpriced
This fixes an issue where local transactions are discarded as
underpriced when the pool and queue are full.
2018-05-02 11:04:40 +03:00
Péter Szilágyi
448d17b8f7 Merge pull request #16630 from tstranex/master
vendor: Fix index out of range panic when size is bigger than 1 TiB
2018-05-02 10:58:14 +03:00
Péter Szilágyi
9922943b42 Merge pull request #16636 from reductionista/travis
travis.yml: remove obsolete brew-cask install
2018-05-02 10:41:50 +03:00
timothy
a1949d0788 vendor: fix leveldb crash when bigger than 1 TiB 2018-05-02 10:34:31 +03:00
Eli
9f6af6f812 whisper: Golint fixes in whisper packages (#16637) 2018-05-02 08:17:17 +02:00
Domino Valdano
ea171d5bd9 travis.yml: remove obsolete brew-cask install 2018-05-01 11:41:34 -07:00
Péter Szilágyi
1da33028ce Merge pull request #16588 from karalabe/tracer-dirty-fix
core, eth: fix tracer dirty finalization
2018-04-27 16:28:19 +03:00
Péter Szilágyi
7a7428a027 core, eth: fix tracer dirty finalization 2018-04-27 14:29:18 +03:00
xincaosu
cfe8f5fd94 trie: remove unused buf parameter (#16583) 2018-04-27 12:45:02 +03:00
Martin Klepsch
852aa143ac cmd/utils: point users to --syncmode under DEPRECATED (#16572)
Indicate that --light and --fast options are replaced by --syncmode
2018-04-27 12:32:06 +03:00
Felix Lange
b724d1aada core/state: cache missing storage entries (#16584) 2018-04-27 12:13:23 +03:00
kimmylin
86be91b3e2 core/types: avoid duplicating transactions on changing signer (#16435) 2018-04-24 10:39:03 +03:00
Felix Lange
e7067be94f cmd/geth, mobile: add memsize to pprof server (#16532)
* cmd/geth, mobile: add memsize to pprof server

This is a temporary change, to be reverted before the next release.

* cmd/geth: fix variable name
2018-04-23 16:20:39 +03:00
Péter Szilágyi
9586f2acc7 VERSION, params: begin release cycle 1.8.7 2018-04-23 15:54:11 +03:00
Péter Szilágyi
12683feca7 params: release v1.8.6 to fix docker images 2018-04-23 15:52:02 +03:00
Péter Szilágyi
49371bf255 Dockerfile: drop legacy discovery v5 port mappings 2018-04-23 15:51:30 +03:00
Péter Szilágyi
16a78b095e Merge pull request #16552 from karalabe/revert-docker-user
Dockerfile: revert the user change PR that broke all APIs
2018-04-23 15:27:24 +03:00
Péter Szilágyi
96a6c8ba0a Dockerfile: revert the user change PR that broke all APIs 2018-04-23 15:26:15 +03:00
Péter Szilágyi
7d2c730acb Merge pull request #16551 from ethereum/revert-16477-puppeth-dockerfile-permission-fix
Revert "cmd/puppeth: fix node deploys for updated dockerfile user"
2018-04-23 15:24:17 +03:00
Péter Szilágyi
abd881f6d4 Revert "cmd/puppeth: fix node deploys for updated dockerfile user" 2018-04-23 15:23:56 +03:00
Péter Szilágyi
4f91831aec Merge pull request #16550 from ethereum/revert-16478-fix-alltools-dockerfile
Revert "Dockerfile.alltools: fix invalid command"
2018-04-23 15:23:47 +03:00
Péter Szilágyi
3f2583d6d1 Revert "Dockerfile.alltools: fix invalid command" 2018-04-23 15:23:14 +03:00
Vie
26a4dbb467 cmd/geth: update the copyright year in the geth command usage (#16537) 2018-04-23 13:45:43 +03:00
Péter Szilágyi
50aa1dcfda VERSION, params: begin Geth 1.8.6 release cycle 2018-04-23 10:06:45 +03:00
61 changed files with 1311 additions and 260 deletions

View File

@@ -31,7 +31,6 @@ matrix:
script:
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- brew update
- brew install caskroom/cask/brew-cask
- brew cask install osxfuse
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES

View File

@@ -12,11 +12,5 @@ FROM alpine:latest
RUN apk add --no-cache ca-certificates
COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
RUN addgroup -g 1000 geth && \
adduser -h /root -D -u 1000 -G geth geth && \
chown geth:geth /root
USER geth
EXPOSE 8545 8546 30303 30303/udp 30304/udp
EXPOSE 8545 8546 30303 30303/udp
ENTRYPOINT ["geth"]

View File

@@ -12,10 +12,4 @@ FROM alpine:latest
RUN apk add --no-cache ca-certificates
COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
RUN addgroup -g 1000 geth && \
adduser -h /root -D -u 1000 -G geth geth && \
chown geth:geth /root
USER geth
EXPOSE 8545 8546 30303 30303/udp 30304/udp
EXPOSE 8545 8546 30303 30303/udp

View File

@@ -1 +1 @@
1.8.5
1.8.7

View File

@@ -33,15 +33,15 @@ type Event struct {
Inputs Arguments
}
func (event Event) String() string {
inputs := make([]string, len(event.Inputs))
for i, input := range event.Inputs {
func (e Event) String() string {
inputs := make([]string, len(e.Inputs))
for i, input := range e.Inputs {
inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
if input.Indexed {
inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type)
}
}
return fmt.Sprintf("event %v(%v)", event.Name, strings.Join(inputs, ", "))
return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", "))
}
// Id returns the canonical representation of the event's signature used by the

View File

@@ -731,7 +731,7 @@ func doAndroidArchive(cmdline []string) {
// Build the Android archive and Maven resources
build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
if *local {
// If we're building locally, copy bundle to build dir and skip Maven
@@ -852,7 +852,7 @@ func doXCodeFramework(cmdline []string) {
// Build the iOS XCode framework
build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
build.MustRun(gomobileTool("init"))
bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
if *local {
// If we're building locally, use the build folder and stop afterwards

View File

@@ -12,6 +12,11 @@ synchronised with the chain or a particular Ethereum node that has no built-in (
Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
Check out
* the [tutorial](tutorial.md) for some concrete examples on how the signer works.
* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory.
## Command line flags
Clef accepts the following command line options:
@@ -49,7 +54,6 @@ Example:
signer -keystore /my/keystore -chainid 4
```
Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works.
## Security model
@@ -862,3 +866,12 @@ A UI should conform to the following rules.
along with the UI.
### UI Implementations
There are a couple of implementation for a UI. We'll try to keep this list up to date.
| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |
| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)|
| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: |
| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: |

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

View File

@@ -0,0 +1,23 @@
"""
This implements a dispatcher which listens to localhost:8550, and proxies
requests via qrexec to the service qubes.EthSign on a target domain
"""
import http.server
import socketserver,subprocess
PORT=8550
TARGET_DOMAIN= 'debian-work'
class Dispatcher(http.server.BaseHTTPRequestHandler):
def do_POST(self):
post_data = self.rfile.read(int(self.headers['Content-Length']))
p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
output = p.communicate(post_data)[0]
self.wfile.write(output)
with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
print("Serving at port", PORT)
httpd.serve_forever()

View File

@@ -0,0 +1,16 @@
#!/bin/bash
SIGNER_BIN="/home/user/tools/clef/clef"
SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
# Start clef if not already started
if [ ! -S /home/user/.clef/clef.ipc ]; then
$SIGNER_CMD &
sleep 1
fi
# Should be started by now
if [ -S /home/user/.clef/clef.ipc ]; then
# Post incoming request to HTTP channel
curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
fi

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

198
cmd/clef/docs/setup.md Normal file
View File

@@ -0,0 +1,198 @@
# Setting up Clef
This document describes how Clef can be used in a more secure manner than executing it from your everyday laptop,
in order to ensure that the keys remain safe in the event that your computer should get compromised.
## Qubes OS
### Background
The Qubes operating system is based around virtual machines (qubes), where a set of virtual machines are configured, typically for
different purposes such as:
- personal
- Your personal email, browsing etc
- work
- Work email etc
- vault
- a VM without network access, where gpg-keys and/or keepass credentials are stored.
A couple of dedicated virtual machines handle externalities:
- sys-net provides networking to all other (network-enabled) machines
- sys-firewall handles firewall rules
- sys-usb handles USB devices, and can map usb-devices to certain qubes.
The goal of this document is to describe how we can set up clef to provide secure transaction
signing from a `vault` vm, to another networked qube which runs Dapps.
### Setup
There are two ways that this can be achieved: integrated via Qubes or integrated via networking.
#### 1. Qubes Integrated
Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request
to another qube. The OS then asks the user if the call is permitted.
![Example](qubes/qrexec-example.png)
A policy-file can be created to allow such interaction. On the `target` domain, a service is invoked which can read the
`stdin` from the `client` qube.
This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented. We can set up Clef the same way:
##### Server
![Clef via qrexec](qubes/clef_qubes_qrexec.png)
On the `target` qubes, we need to define the rpc service.
[qubes.Clefsign](qubes/qubes.Clefsign):
```bash
#!/bin/bash
SIGNER_BIN="/home/user/tools/clef/clef"
SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
# Start clef if not already started
if [ ! -S /home/user/.clef/clef.ipc ]; then
$SIGNER_CMD &
sleep 1
fi
# Should be started by now
if [ -S /home/user/.clef/clef.ipc ]; then
# Post incoming request to HTTP channel
curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
fi
```
This RPC service is not complete (see notes about HTTP headers below), but works as a proof-of-concept.
It will forward the data received on `stdin` (forwarded by the OS) to Clef's HTTP channel.
It would have been possible to send data directly to the `/home/user/.clef/.clef.ipc`
socket via e.g `nc -U /home/user/.clef/clef.ipc`, but the reason for sending the request
data over `HTTP` instead of `IPC` is that we want the ability to forward `HTTP` headers.
To enable the service:
``` bash
sudo cp qubes.Clefsign /etc/qubes-rpc/
sudo chmod +x /etc/qubes-rpc/ qubes.Clefsign
```
This setup uses [gtksigner](https://github.com/holiman/gtksigner), which is a very minimal GTK-based UI that works well
with minimal requirements.
##### Client
On the `client` qube, we need to create a listener which will receive the request from the Dapp, and proxy it.
[qubes-client.py](qubes/client/qubes-client.py):
```python
"""
This implements a dispatcher which listens to localhost:8550, and proxies
requests via qrexec to the service qubes.EthSign on a target domain
"""
import http.server
import socketserver,subprocess
PORT=8550
TARGET_DOMAIN= 'debian-work'
class Dispatcher(http.server.BaseHTTPRequestHandler):
def do_POST(self):
post_data = self.rfile.read(int(self.headers['Content-Length']))
p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
output = p.communicate(post_data)[0]
self.wfile.write(output)
with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
print("Serving at port", PORT)
httpd.serve_forever()
```
#### Testing
To test the flow, if we have set up `debian-work` as the `target`, we can do
```bash
$ cat newaccnt.json
{ "id": 0, "jsonrpc": "2.0","method": "account_new","params": []}
$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign
```
This should pop up first a dialog to allow the IPC call:
![one](qubes/qubes_newaccount-1.png)
Followed by a GTK-dialog to approve the operation
![two](qubes/qubes_newaccount-2.png)
To test the full flow, we use the client wrapper. Start it on the `client` qube:
```
[user@work qubes]$ python3 qubes-client.py
```
Make the request over http (`client` qube):
```
[user@work clef]$ cat newaccnt.json | curl -X POST -d @- http://localhost:8550
```
And it should show the same popups again.
##### Pros and cons
The benefits of this setup are:
- This is the qubes-os intended model for inter-qube communication,
- and thus benefits from qubes-os dialogs and policies for user approval
However, it comes with a couple of drawbacks:
- The `qubes-gpg-client` must forward the http request via RPC to the `target` qube. When doing so, the proxy
will either drop important headers, or replace them.
- The `Host` header is most likely `localhost`
- The `Origin` header must be forwarded
- Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header,
since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address.
- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups:
- One qubes-specific where the user specifies the `target` vm
- One clef-specific to approve the transaction
#### 2. Network integrated
The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible
form other qubes.
![Clef via http](qubes/clef_qubes_http.png)
## USBArmory
The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 Mhz ARM processor. It is a pocket-size
computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network
to your computer. Over this new network interface, you can SSH into the device.
Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only
ever connects to a local network between your computer and the device itself.
Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access
to the USB Armory would trivially be able to extract the contents of the device filesystem.

View File

@@ -21,12 +21,12 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"os"
goruntime "runtime"
"runtime/pprof"
"time"
goruntime "runtime"
"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
@@ -86,6 +86,7 @@ func runCmd(ctx *cli.Context) error {
chainConfig *params.ChainConfig
sender = common.BytesToAddress([]byte("sender"))
receiver = common.BytesToAddress([]byte("receiver"))
blockNumber uint64
)
if ctx.GlobalBool(MachineFlag.Name) {
tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -101,6 +102,7 @@ func runCmd(ctx *cli.Context) error {
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
chainConfig = gen.Config
blockNumber = gen.Number
} else {
db, _ := ethdb.NewMemDatabase()
statedb, _ = state.New(common.Hash{}, state.NewDatabase(db))
@@ -156,11 +158,12 @@ func runCmd(ctx *cli.Context) error {
initialGas := ctx.GlobalUint64(GasFlag.Name)
runtimeConfig := runtime.Config{
Origin: sender,
State: statedb,
GasLimit: initialGas,
GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
Value: utils.GlobalBig(ctx, ValueFlag.Name),
Origin: sender,
State: statedb,
GasLimit: initialGas,
GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
Value: utils.GlobalBig(ctx, ValueFlag.Name),
BlockNumber: new(big.Int).SetUint64(blockNumber),
EVMConfig: vm.Config{
Tracer: tracer,
Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),

View File

@@ -146,7 +146,7 @@ func init() {
// Initialize the CLI app and start Geth
app.Action = geth
app.HideVersion = true // we have a command to print the version
app.Copyright = "Copyright 2013-2017 The go-ethereum Authors"
app.Copyright = "Copyright 2013-2018 The go-ethereum Authors"
app.Commands = []cli.Command{
// See chaincmd.go:
initCommand,
@@ -223,6 +223,8 @@ func geth(ctx *cli.Context) error {
// it unlocks any requested accounts, and starts the RPC/IPC interfaces and the
// miner.
func startNode(ctx *cli.Context, stack *node.Node) {
debug.Memsize.Add("node", stack)
// Start up the node itself
utils.StartNode(stack)

View File

@@ -33,7 +33,7 @@ import (
var AppHelpTemplate = `NAME:
{{.App.Name}} - {{.App.Usage}}
Copyright 2013-2017 The go-ethereum Authors
Copyright 2013-2018 The go-ethereum Authors
USAGE:
{{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}

View File

@@ -40,11 +40,11 @@ ADD genesis.json /genesis.json
ADD signer.pass /signer.pass
{{end}}
RUN \
echo 'geth --cache 512 init /genesis.json' > /root/geth.sh && \{{if .Unlock}}
echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> /root/geth.sh && \{{end}}
echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> /root/geth.sh
echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
ENTRYPOINT ["/bin/sh", "/root/geth.sh"]
ENTRYPOINT ["/bin/sh", "geth.sh"]
`
// nodeComposefile is the docker-compose.yml file required to deploy and maintain

View File

@@ -158,11 +158,11 @@ var (
}
FastSyncFlag = cli.BoolFlag{
Name: "fast",
Usage: "Enable fast syncing through state downloads",
Usage: "Enable fast syncing through state downloads (replaced by --syncmode)",
}
LightModeFlag = cli.BoolFlag{
Name: "light",
Usage: "Enable light client mode",
Usage: "Enable light client mode (replaced by --syncmode)",
}
defaultSyncMode = eth.DefaultConfig.SyncMode
SyncModeFlag = TextMarshalerFlag{

View File

@@ -178,9 +178,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash {
}
value.SetBytes(content)
}
if (value != common.Hash{}) {
self.cachedStorage[key] = value
}
self.cachedStorage[key] = value
return value
}
@@ -197,7 +195,6 @@ func (self *stateObject) SetState(db Database, key, value common.Hash) {
func (self *stateObject) setState(key, value common.Hash) {
self.cachedStorage[key] = value
self.dirtyStorage[key] = value
}
// updateTrie writes cached storage modifications into the object's storage trie.

View File

@@ -572,27 +572,6 @@ func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) {
self.txIndex = ti
}
// DeleteSuicides flags the suicided objects for deletion so that it
// won't be referenced again when called / queried up on.
//
// DeleteSuicides should not be used for consensus related updates
// under any circumstances.
func (s *StateDB) DeleteSuicides() {
// Reset refund so that any used-gas calculations can use this method.
s.clearJournalAndRefund()
for addr := range s.stateObjectsDirty {
stateObject := s.stateObjects[addr]
// If the object has been removed by a suicide
// flag the object as deleted.
if stateObject.suicided {
stateObject.deleted = true
}
delete(s.stateObjectsDirty, addr)
}
}
func (s *StateDB) clearJournalAndRefund() {
s.journal = newJournal()
s.validRevisions = s.validRevisions[:0]

View File

@@ -618,7 +618,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
// If the transaction pool is full, discard underpriced transactions
if uint64(len(pool.all)) >= pool.config.GlobalSlots+pool.config.GlobalQueue {
// If the new transaction is underpriced, don't accept it
if pool.priced.Underpriced(tx, pool.locals) {
if !local && pool.priced.Underpriced(tx, pool.locals) {
log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice())
underpricedTxCounter.Inc(1)
return false, ErrUnderpriced

View File

@@ -1346,7 +1346,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
defer sub.Unsubscribe()
// Create a number of test accounts and fund them
keys := make([]*ecdsa.PrivateKey, 3)
keys := make([]*ecdsa.PrivateKey, 4)
for i := 0; i < len(keys); i++ {
keys[i], _ = crypto.GenerateKey()
pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
@@ -1406,18 +1406,22 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
t.Fatalf("pool internal state corrupted: %v", err)
}
// Ensure that adding local transactions can push out even higher priced ones
tx := pricedTransaction(1, 100000, big.NewInt(0), keys[2])
if err := pool.AddLocal(tx); err != nil {
t.Fatalf("failed to add underpriced local transaction: %v", err)
ltx = pricedTransaction(1, 100000, big.NewInt(0), keys[2])
if err := pool.AddLocal(ltx); err != nil {
t.Fatalf("failed to append underpriced local transaction: %v", err)
}
ltx = pricedTransaction(0, 100000, big.NewInt(0), keys[3])
if err := pool.AddLocal(ltx); err != nil {
t.Fatalf("failed to add new underpriced local transaction: %v", err)
}
pending, queued = pool.Stats()
if pending != 2 {
t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2)
if pending != 3 {
t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3)
}
if queued != 2 {
t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
if queued != 1 {
t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1)
}
if err := validateEvents(events, 1); err != nil {
if err := validateEvents(events, 2); err != nil {
t.Fatalf("local event firing failed: %v", err)
}
if err := validateTxPoolInternals(pool); err != nil {

View File

@@ -339,11 +339,14 @@ type TransactionsByPriceAndNonce struct {
func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce {
// Initialize a price based heap with the head transactions
heads := make(TxByPrice, 0, len(txs))
for _, accTxs := range txs {
for from, accTxs := range txs {
heads = append(heads, accTxs[0])
// Ensure the sender address is from the signer
acc, _ := Sender(signer, accTxs[0])
txs[acc] = accTxs[1:]
if from != acc {
delete(txs, from)
}
}
heap.Init(&heads)

View File

@@ -139,15 +139,15 @@ func (c *Contract) Value() *big.Int {
}
// SetCode sets the code to the contract
func (self *Contract) SetCode(hash common.Hash, code []byte) {
self.Code = code
self.CodeHash = hash
func (c *Contract) SetCode(hash common.Hash, code []byte) {
c.Code = code
c.CodeHash = hash
}
// SetCallCode sets the code of the contract and address of the backing data
// object
func (self *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) {
self.Code = code
self.CodeHash = hash
self.CodeAddr = addr
func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) {
c.Code = code
c.CodeHash = hash
c.CodeAddr = addr
}

View File

@@ -160,6 +160,11 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
precompiles = PrecompiledContractsByzantium
}
if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 {
// Calling a non existing account, don't do antything, but ping the tracer
if evm.vmConfig.Debug && evm.depth == 0 {
evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value)
evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil)
}
return nil, gas, nil
}
evm.StateDB.CreateAccount(addr)

View File

@@ -31,9 +31,9 @@ import (
type Storage map[common.Hash]common.Hash
func (self Storage) Copy() Storage {
func (s Storage) Copy() Storage {
cpy := make(Storage)
for key, value := range self {
for key, value := range s {
cpy[key] = value
}

View File

@@ -51,14 +51,14 @@ func (m *Memory) Resize(size uint64) {
}
// Get returns offset + size as a new slice
func (self *Memory) Get(offset, size int64) (cpy []byte) {
func (m *Memory) Get(offset, size int64) (cpy []byte) {
if size == 0 {
return nil
}
if len(self.store) > int(offset) {
if len(m.store) > int(offset) {
cpy = make([]byte, size)
copy(cpy, self.store[offset:offset+size])
copy(cpy, m.store[offset:offset+size])
return
}
@@ -67,13 +67,13 @@ func (self *Memory) Get(offset, size int64) (cpy []byte) {
}
// GetPtr returns the offset + size
func (self *Memory) GetPtr(offset, size int64) []byte {
func (m *Memory) GetPtr(offset, size int64) []byte {
if size == 0 {
return nil
}
if len(self.store) > int(offset) {
return self.store[offset : offset+size]
if len(m.store) > int(offset) {
return m.store[offset : offset+size]
}
return nil

View File

@@ -375,10 +375,10 @@ var opCodeToString = map[OpCode]string{
SWAP: "SWAP",
}
func (o OpCode) String() string {
str := opCodeToString[o]
func (op OpCode) String() string {
str := opCodeToString[op]
if len(str) == 0 {
return fmt.Sprintf("Missing opcode 0x%x", int(o))
return fmt.Sprintf("Missing opcode 0x%x", int(op))
}
return str

View File

@@ -201,7 +201,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err)
break
}
task.statedb.DeleteSuicides()
task.statedb.Finalise(true)
task.results[i] = &txTraceResult{Result: res}
}
// Stream the result back to the user or abort on teardown
@@ -640,7 +640,8 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree
if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil {
return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err)
}
statedb.DeleteSuicides()
// Ensure any modifications are committed to the state
statedb.Finalise(true)
}
return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash)
}

View File

@@ -28,10 +28,13 @@ import (
"github.com/ethereum/go-ethereum/log/term"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/metrics/exp"
"github.com/fjl/memsize/memsizeui"
colorable "github.com/mattn/go-colorable"
"gopkg.in/urfave/cli.v1"
)
var Memsize memsizeui.Handler
var (
verbosityFlag = cli.IntFlag{
Name: "verbosity",
@@ -129,21 +132,25 @@ func Setup(ctx *cli.Context) error {
// pprof server
if ctx.GlobalBool(pprofFlag.Name) {
// Hook go-metrics into expvar on any /debug/metrics request, load all vars
// from the registry into expvar, and execute regular expvar handler.
exp.Exp(metrics.DefaultRegistry)
address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name))
go func() {
log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address))
if err := http.ListenAndServe(address, nil); err != nil {
log.Error("Failure in running pprof server", "err", err)
}
}()
StartPProf(address)
}
return nil
}
func StartPProf(address string) {
// Hook go-metrics into expvar on any /debug/metrics request, load all vars
// from the registry into expvar, and execute regular expvar handler.
exp.Exp(metrics.DefaultRegistry)
http.Handle("/memsize/", http.StripPrefix("/memsize", &Memsize))
log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address))
go func() {
if err := http.ListenAndServe(address, nil); err != nil {
log.Error("Failure in running pprof server", "err", err)
}
}()
}
// Exit stops all running profiles, flushing their output to the
// respective file.
func Exit() {

View File

@@ -102,8 +102,8 @@ func randomSource() *rand.Rand {
// call the functions of the otto vm directly to circumvent the queue. These
// functions should be used if and only if running a routine that was already
// called from JS through an RPC call.
func (self *JSRE) runEventLoop() {
defer close(self.closed)
func (re *JSRE) runEventLoop() {
defer close(re.closed)
vm := otto.New()
r := randomSource()
@@ -202,14 +202,14 @@ loop:
break loop
}
}
case req := <-self.evalQueue:
case req := <-re.evalQueue:
// run the code, send the result back
req.fn(vm)
close(req.done)
if waitForCallbacks && (len(registry) == 0) {
break loop
}
case waitForCallbacks = <-self.stopEventLoop:
case waitForCallbacks = <-re.stopEventLoop:
if !waitForCallbacks || (len(registry) == 0) {
break loop
}
@@ -223,31 +223,31 @@ loop:
}
// Do executes the given function on the JS event loop.
func (self *JSRE) Do(fn func(*otto.Otto)) {
func (re *JSRE) Do(fn func(*otto.Otto)) {
done := make(chan bool)
req := &evalReq{fn, done}
self.evalQueue <- req
re.evalQueue <- req
<-done
}
// stops the event loop before exit, optionally waits for all timers to expire
func (self *JSRE) Stop(waitForCallbacks bool) {
func (re *JSRE) Stop(waitForCallbacks bool) {
select {
case <-self.closed:
case self.stopEventLoop <- waitForCallbacks:
<-self.closed
case <-re.closed:
case re.stopEventLoop <- waitForCallbacks:
<-re.closed
}
}
// Exec(file) loads and runs the contents of a file
// if a relative path is given, the jsre's assetPath is used
func (self *JSRE) Exec(file string) error {
code, err := ioutil.ReadFile(common.AbsolutePath(self.assetPath, file))
func (re *JSRE) Exec(file string) error {
code, err := ioutil.ReadFile(common.AbsolutePath(re.assetPath, file))
if err != nil {
return err
}
var script *otto.Script
self.Do(func(vm *otto.Otto) {
re.Do(func(vm *otto.Otto) {
script, err = vm.Compile(file, code)
if err != nil {
return
@@ -259,36 +259,36 @@ func (self *JSRE) Exec(file string) error {
// Bind assigns value v to a variable in the JS environment
// This method is deprecated, use Set.
func (self *JSRE) Bind(name string, v interface{}) error {
return self.Set(name, v)
func (re *JSRE) Bind(name string, v interface{}) error {
return re.Set(name, v)
}
// Run runs a piece of JS code.
func (self *JSRE) Run(code string) (v otto.Value, err error) {
self.Do(func(vm *otto.Otto) { v, err = vm.Run(code) })
func (re *JSRE) Run(code string) (v otto.Value, err error) {
re.Do(func(vm *otto.Otto) { v, err = vm.Run(code) })
return v, err
}
// Get returns the value of a variable in the JS environment.
func (self *JSRE) Get(ns string) (v otto.Value, err error) {
self.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) })
func (re *JSRE) Get(ns string) (v otto.Value, err error) {
re.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) })
return v, err
}
// Set assigns value v to a variable in the JS environment.
func (self *JSRE) Set(ns string, v interface{}) (err error) {
self.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) })
func (re *JSRE) Set(ns string, v interface{}) (err error) {
re.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) })
return err
}
// loadScript executes a JS script from inside the currently executing JS code.
func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value {
func (re *JSRE) loadScript(call otto.FunctionCall) otto.Value {
file, err := call.Argument(0).ToString()
if err != nil {
// TODO: throw exception
return otto.FalseValue()
}
file = common.AbsolutePath(self.assetPath, file)
file = common.AbsolutePath(re.assetPath, file)
source, err := ioutil.ReadFile(file)
if err != nil {
// TODO: throw exception
@@ -305,10 +305,10 @@ func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value {
// Evaluate executes code and pretty prints the result to the specified output
// stream.
func (self *JSRE) Evaluate(code string, w io.Writer) error {
func (re *JSRE) Evaluate(code string, w io.Writer) error {
var fail error
self.Do(func(vm *otto.Otto) {
re.Do(func(vm *otto.Otto) {
val, err := vm.Run(code)
if err != nil {
prettyError(vm, err, w)
@@ -321,8 +321,8 @@ func (self *JSRE) Evaluate(code string, w io.Writer) error {
}
// Compile compiles and then runs a piece of JS code.
func (self *JSRE) Compile(filename string, src interface{}) (err error) {
self.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) })
func (re *JSRE) Compile(filename string, src interface{}) (err error) {
re.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) })
return err
}

View File

@@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/ethstats"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/les"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
@@ -72,6 +73,9 @@ type NodeConfig struct {
// WhisperEnabled specifies whether the node should run the Whisper protocol.
WhisperEnabled bool
// Listening address of pprof server.
PprofAddress string
}
// defaultNodeConfig contains the default node configuration values to use if all
@@ -107,6 +111,11 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) {
if config.BootstrapNodes == nil || config.BootstrapNodes.Size() == 0 {
config.BootstrapNodes = defaultNodeConfig.BootstrapNodes
}
if config.PprofAddress != "" {
debug.StartPProf(config.PprofAddress)
}
// Create the empty networking stack
nodeConf := &node.Config{
Name: clientIdentifier,
@@ -127,6 +136,8 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) {
return nil, err
}
debug.Memsize.Add("node", rawStack)
var genesis *core.Genesis
if config.EthereumGenesis != "" {
// Parse the user supplied genesis spec if not mainnet

View File

@@ -23,7 +23,7 @@ import (
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 8 // Minor version component of the current release
VersionPatch = 5 // Patch version component of the current release
VersionPatch = 7 // Patch version component of the current release
VersionMeta = "stable" // Version metadata to append to the version string
)

View File

@@ -104,7 +104,7 @@ func (t *BlockTest) Run() error {
return err
}
if gblock.Hash() != t.json.Genesis.Hash {
return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x\n", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6])
return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6])
}
if gblock.Root() != t.json.Genesis.StateRoot {
return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes()[:6], t.json.Genesis.StateRoot[:6])

View File

@@ -23,7 +23,7 @@ import (
"github.com/ethereum/go-ethereum/params"
)
// This table defines supported forks and their chain config.
// Forks table defines supported forks and their chain config.
var Forks = map[string]*params.ChainConfig{
"Frontier": {
ChainId: big.NewInt(1),

View File

@@ -42,7 +42,7 @@ var (
difficultyTestDir = filepath.Join(baseDir, "BasicTests")
)
func readJson(reader io.Reader, value interface{}) error {
func readJSON(reader io.Reader, value interface{}) error {
data, err := ioutil.ReadAll(reader)
if err != nil {
return fmt.Errorf("error reading JSON file: %v", err)
@@ -57,14 +57,14 @@ func readJson(reader io.Reader, value interface{}) error {
return nil
}
func readJsonFile(fn string, value interface{}) error {
func readJSONFile(fn string, value interface{}) error {
file, err := os.Open(fn)
if err != nil {
return err
}
defer file.Close()
err = readJson(file, value)
err = readJSON(file, value)
if err != nil {
return fmt.Errorf("%s in file %s", err.Error(), fn)
}
@@ -169,9 +169,8 @@ func (tm *testMatcher) checkFailure(t *testing.T, name string, err error) error
if err != nil {
t.Logf("error: %v", err)
return nil
} else {
return fmt.Errorf("test succeeded unexpectedly")
}
return fmt.Errorf("test succeeded unexpectedly")
}
return err
}
@@ -213,7 +212,7 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte
// Load the file as map[string]<testType>.
m := makeMapFromTestFunc(runTest)
if err := readJsonFile(path, m.Addr().Interface()); err != nil {
if err := readJSONFile(path, m.Addr().Interface()); err != nil {
t.Fatal(err)
}

View File

@@ -72,9 +72,8 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error {
if err := rlp.DecodeBytes(tt.json.RLP, tx); err != nil {
if tt.json.Transaction == nil {
return nil
} else {
return fmt.Errorf("RLP decoding failed: %v", err)
}
return fmt.Errorf("RLP decoding failed: %v", err)
}
// Check sender derivation.
signer := types.MakeSigner(config, new(big.Int).SetUint64(uint64(tt.json.BlockNumber)))

View File

@@ -303,7 +303,7 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path []
it.path = path
it.stack = append(it.stack, state)
if parentIndex != nil {
*parentIndex += 1
*parentIndex++
}
}
@@ -380,7 +380,7 @@ func (it *differenceIterator) Next(bool) bool {
if !it.b.Next(true) {
return false
}
it.count += 1
it.count++
if it.eof {
// a has reached eof, so we just return all elements from b
@@ -395,7 +395,7 @@ func (it *differenceIterator) Next(bool) bool {
it.eof = true
return true
}
it.count += 1
it.count++
case 1:
// b is before a
return true
@@ -405,12 +405,12 @@ func (it *differenceIterator) Next(bool) bool {
if !it.b.Next(hasHash) {
return false
}
it.count += 1
it.count++
if !it.a.Next(hasHash) {
it.eof = true
return true
}
it.count += 1
it.count++
}
}
}
@@ -504,14 +504,14 @@ func (it *unionIterator) Next(descend bool) bool {
skipped := heap.Pop(it.items).(NodeIterator)
// Skip the whole subtree if the nodes have hashes; otherwise just skip this node
if skipped.Next(skipped.Hash() == common.Hash{}) {
it.count += 1
it.count++
// If there are more elements, push the iterator back on the heap
heap.Push(it.items, skipped)
}
}
if least.Next(descend) {
it.count += 1
it.count++
heap.Push(it.items, least)
}

View File

@@ -123,17 +123,17 @@ func decodeNode(hash, buf []byte, cachegen uint16) (node, error) {
}
switch c, _ := rlp.CountValues(elems); c {
case 2:
n, err := decodeShort(hash, buf, elems, cachegen)
n, err := decodeShort(hash, elems, cachegen)
return n, wrapError(err, "short")
case 17:
n, err := decodeFull(hash, buf, elems, cachegen)
n, err := decodeFull(hash, elems, cachegen)
return n, wrapError(err, "full")
default:
return nil, fmt.Errorf("invalid number of list elements: %v", c)
}
}
func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) {
func decodeShort(hash, elems []byte, cachegen uint16) (node, error) {
kbuf, rest, err := rlp.SplitString(elems)
if err != nil {
return nil, err
@@ -155,7 +155,7 @@ func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) {
return &shortNode{key, r, flag}, nil
}
func decodeFull(hash, buf, elems []byte, cachegen uint16) (*fullNode, error) {
func decodeFull(hash, elems []byte, cachegen uint16) (*fullNode, error) {
n := &fullNode{flags: nodeFlag{hash: hash, gen: cachegen}}
for i := 0; i < 16; i++ {
cld, rest, err := decodeRef(elems, cachegen)

21
vendor/github.com/fjl/memsize/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018 Felix Lange
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

119
vendor/github.com/fjl/memsize/bitmap.go generated vendored Normal file
View File

@@ -0,0 +1,119 @@
package memsize
import (
"math/bits"
)
const (
uintptrBits = 32 << (uint64(^uintptr(0)) >> 63)
uintptrBytes = uintptrBits / 8
bmBlockRange = 1 * 1024 * 1024 // bytes covered by bmBlock
bmBlockWords = bmBlockRange / uintptrBits
)
// bitmap is a sparse bitmap.
type bitmap struct {
blocks map[uintptr]*bmBlock
}
func newBitmap() *bitmap {
return &bitmap{make(map[uintptr]*bmBlock)}
}
// markRange sets n consecutive bits starting at addr.
func (b *bitmap) markRange(addr, n uintptr) {
for end := addr + n; addr < end; {
block, baddr := b.block(addr)
for i := baddr; i < bmBlockRange && addr < end; i++ {
block.mark(i)
addr++
}
}
}
// isMarked returns the value of the bit at the given address.
func (b *bitmap) isMarked(addr uintptr) bool {
block, baddr := b.block(addr)
return block.isMarked(baddr)
}
// countRange returns the number of set bits in the range (addr,addr+n).
func (b *bitmap) countRange(addr, n uintptr) uintptr {
c := uintptr(0)
for end := addr + n; addr < end; {
block, baddr := b.block(addr)
bend := uintptr(bmBlockRange - 1)
if baddr+(end-addr) < bmBlockRange {
bend = baddr + (end - addr)
}
c += uintptr(block.count(baddr, bend))
// Move addr to next block.
addr += bmBlockRange - baddr
}
return c
}
// block finds the block corresponding to the given memory address.
// It also returns the block's starting address.
func (b *bitmap) block(addr uintptr) (*bmBlock, uintptr) {
index := addr / bmBlockRange
block := b.blocks[index]
if block == nil {
block = new(bmBlock)
b.blocks[index] = block
}
return block, addr % bmBlockRange
}
// size returns the sum of the byte sizes of all blocks.
func (b *bitmap) size() uintptr {
return uintptr(len(b.blocks)) * bmBlockWords * uintptrBytes
}
// utilization returns the mean percentage of one bits across all blocks.
func (b *bitmap) utilization() float32 {
var avg float32
for _, block := range b.blocks {
avg += float32(block.count(0, bmBlockRange-1)) / float32(bmBlockRange)
}
return avg / float32(len(b.blocks))
}
// bmBlock is a bitmap block.
type bmBlock [bmBlockWords]uintptr
// mark sets the i'th bit to one.
func (b *bmBlock) mark(i uintptr) {
b[i/uintptrBits] |= 1 << (i % uintptrBits)
}
// isMarked returns the value of the i'th bit.
func (b *bmBlock) isMarked(i uintptr) bool {
return (b[i/uintptrBits] & (1 << (i % uintptrBits))) != 0
}
// count returns the number of set bits in the range (start,end).
func (b *bmBlock) count(start, end uintptr) (count int) {
br := b[start/uintptrBits : end/uintptrBits+1]
for i, w := range br {
if i == 0 {
w &= blockmask(start)
}
if i == len(br)-1 {
w &^= blockmask(end)
}
count += onesCountPtr(w)
}
return count
}
func blockmask(x uintptr) uintptr {
return ^uintptr(0) << (x % uintptrBits)
}
func onesCountPtr(x uintptr) int {
if uintptrBits == 64 {
return bits.OnesCount64(uint64(x))
}
return bits.OnesCount32(uint32(x))
}

16
vendor/github.com/fjl/memsize/doc.go generated vendored Normal file
View File

@@ -0,0 +1,16 @@
/*
Package memsize computes the size of your object graph.
So you made a spiffy algorithm and it works really well, but geez it's using
way too much memory. Where did it all go? memsize to the rescue!
To get started, find a value that references all your objects and scan it.
This traverses the graph, counting sizes per type.
sizes := memsize.Scan(myValue)
fmt.Println(sizes.Total)
memsize can handle cycles just fine and tracks both private and public struct fields.
Unfortunately function closures cannot be inspected in any way.
*/
package memsize

243
vendor/github.com/fjl/memsize/memsize.go generated vendored Normal file
View File

@@ -0,0 +1,243 @@
package memsize
import (
"bytes"
"fmt"
"reflect"
"sort"
"strings"
"text/tabwriter"
"unsafe"
)
// Scan traverses all objects reachable from v and counts how much memory
// is used per type. The value must be a non-nil pointer to any value.
func Scan(v interface{}) Sizes {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
panic("value to scan must be non-nil pointer")
}
stopTheWorld("memsize scan")
defer startTheWorld()
ctx := newContext()
ctx.scan(invalidAddr, rv, false)
ctx.s.BitmapSize = ctx.seen.size()
ctx.s.BitmapUtilization = ctx.seen.utilization()
return *ctx.s
}
// Sizes is the result of a scan.
type Sizes struct {
Total uintptr
ByType map[reflect.Type]*TypeSize
// Internal stats (for debugging)
BitmapSize uintptr
BitmapUtilization float32
}
type TypeSize struct {
Total uintptr
Count uintptr
}
func newSizes() *Sizes {
return &Sizes{ByType: make(map[reflect.Type]*TypeSize)}
}
// Report returns a human-readable report.
func (s Sizes) Report() string {
type typLine struct {
name string
count uintptr
total uintptr
}
tab := []typLine{{"ALL", 0, s.Total}}
for _, typ := range s.ByType {
tab[0].count += typ.Count
}
maxname := 0
for typ, s := range s.ByType {
line := typLine{typ.String(), s.Count, s.Total}
tab = append(tab, line)
if len(line.name) > maxname {
maxname = len(line.name)
}
}
sort.Slice(tab, func(i, j int) bool { return tab[i].total > tab[j].total })
buf := new(bytes.Buffer)
w := tabwriter.NewWriter(buf, 0, 0, 0, ' ', tabwriter.AlignRight)
for _, line := range tab {
namespace := strings.Repeat(" ", maxname-len(line.name))
fmt.Fprintf(w, "%s%s\t %v\t %s\t\n", line.name, namespace, line.count, HumanSize(line.total))
}
w.Flush()
return buf.String()
}
// addValue is called during scan and adds the memory of given object.
func (s *Sizes) addValue(v reflect.Value, size uintptr) {
s.Total += size
rs := s.ByType[v.Type()]
if rs == nil {
rs = new(TypeSize)
s.ByType[v.Type()] = rs
}
rs.Total += size
rs.Count++
}
type context struct {
// We track previously scanned objects to prevent infinite loops
// when scanning cycles and to prevent counting objects more than once.
seen *bitmap
tc typCache
s *Sizes
}
func newContext() *context {
return &context{seen: newBitmap(), tc: make(typCache), s: newSizes()}
}
// scan walks all objects below v, determining their size. All scan* functions return the
// amount of 'extra' memory (e.g. slice data) that is referenced by the object.
func (c *context) scan(addr address, v reflect.Value, add bool) (extraSize uintptr) {
size := v.Type().Size()
var marked uintptr
if addr.valid() {
marked = c.seen.countRange(uintptr(addr), size)
if marked == size {
return 0 // Skip if we have already seen the whole object.
}
c.seen.markRange(uintptr(addr), size)
}
// fmt.Printf("%v: %v ⮑ (marked %d)\n", addr, v.Type(), marked)
if c.tc.needScan(v.Type()) {
extraSize = c.scanContent(addr, v)
}
// fmt.Printf("%v: %v %d (add %v, size %d, marked %d, extra %d)\n", addr, v.Type(), size+extraSize, add, v.Type().Size(), marked, extraSize)
if add {
size -= marked
size += extraSize
c.s.addValue(v, size)
}
return extraSize
}
func (c *context) scanContent(addr address, v reflect.Value) uintptr {
switch v.Kind() {
case reflect.Array:
return c.scanArray(addr, v)
case reflect.Chan:
return c.scanChan(v)
case reflect.Func:
// can't do anything here
return 0
case reflect.Interface:
return c.scanInterface(v)
case reflect.Map:
return c.scanMap(v)
case reflect.Ptr:
if !v.IsNil() {
c.scan(address(v.Pointer()), v.Elem(), true)
}
return 0
case reflect.Slice:
return c.scanSlice(v)
case reflect.String:
return uintptr(v.Len())
case reflect.Struct:
return c.scanStruct(addr, v)
default:
unhandledKind(v.Kind())
return 0
}
}
func (c *context) scanChan(v reflect.Value) uintptr {
etyp := v.Type().Elem()
extra := uintptr(0)
if c.tc.needScan(etyp) {
// Scan the channel buffer. This is unsafe but doesn't race because
// the world is stopped during scan.
hchan := unsafe.Pointer(v.Pointer())
for i := uint(0); i < uint(v.Cap()); i++ {
addr := chanbuf(hchan, i)
elem := reflect.NewAt(etyp, addr).Elem()
extra += c.scanContent(address(addr), elem)
}
}
return uintptr(v.Cap())*etyp.Size() + extra
}
func (c *context) scanStruct(base address, v reflect.Value) uintptr {
extra := uintptr(0)
for i := 0; i < v.NumField(); i++ {
f := v.Type().Field(i)
if c.tc.needScan(f.Type) {
addr := base.addOffset(f.Offset)
extra += c.scanContent(addr, v.Field(i))
}
}
return extra
}
func (c *context) scanArray(addr address, v reflect.Value) uintptr {
esize := v.Type().Elem().Size()
extra := uintptr(0)
for i := 0; i < v.Len(); i++ {
extra += c.scanContent(addr, v.Index(i))
addr = addr.addOffset(esize)
}
return extra
}
func (c *context) scanSlice(v reflect.Value) uintptr {
slice := v.Slice(0, v.Cap())
esize := slice.Type().Elem().Size()
base := slice.Pointer()
// Add size of the unscanned portion of the backing array to extra.
blen := uintptr(slice.Len()) * esize
marked := c.seen.countRange(base, blen)
extra := blen - marked
c.seen.markRange(uintptr(base), blen)
if c.tc.needScan(slice.Type().Elem()) {
// Elements may contain pointers, scan them individually.
addr := address(base)
for i := 0; i < slice.Len(); i++ {
extra += c.scanContent(addr, slice.Index(i))
addr = addr.addOffset(esize)
}
}
return extra
}
func (c *context) scanMap(v reflect.Value) uintptr {
var (
typ = v.Type()
len = uintptr(v.Len())
extra = uintptr(0)
)
if c.tc.needScan(typ.Key()) || c.tc.needScan(typ.Elem()) {
for _, k := range v.MapKeys() {
extra += c.scan(invalidAddr, k, false)
extra += c.scan(invalidAddr, v.MapIndex(k), false)
}
}
return len*typ.Key().Size() + len*typ.Elem().Size() + extra
}
func (c *context) scanInterface(v reflect.Value) uintptr {
elem := v.Elem()
if !elem.IsValid() {
return 0 // nil interface
}
c.scan(invalidAddr, elem, false)
if !c.tc.isPointer(elem.Type()) {
// Account for non-pointer size of the value.
return elem.Type().Size()
}
return 0
}

106
vendor/github.com/fjl/memsize/memsizeui/template.go generated vendored Normal file
View File

@@ -0,0 +1,106 @@
package memsizeui
import (
"html/template"
"strconv"
"sync"
"github.com/fjl/memsize"
)
var (
base *template.Template // the "base" template
baseInitOnce sync.Once
)
func baseInit() {
base = template.Must(template.New("base").Parse(`<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>memsize</title>
<style>
body {
font-family: sans-serif;
}
button, .button {
display: inline-block;
font-weight: bold;
color: black;
text-decoration: none;
font-size: inherit;
padding: 3pt;
margin: 3pt;
background-color: #eee;
border: 1px solid #999;
border-radius: 2pt;
}
form.inline {
display: inline-block;
}
</style>
</head>
<body>
{{template "content" .}}
</body>
</html>`))
base.Funcs(template.FuncMap{
"quote": strconv.Quote,
"humansize": memsize.HumanSize,
})
template.Must(base.New("rootbuttons").Parse(`
<a class="button" href="{{$.Link ""}}">Overview</a>
{{- range $root := .Roots -}}
<form class="inline" method="POST" action="{{$.Link "scan?root=" $root}}">
<button type="submit">Scan {{quote $root}}</button>
</form>
{{- end -}}`))
}
func contentTemplate(source string) *template.Template {
baseInitOnce.Do(baseInit)
t := template.Must(base.Clone())
template.Must(t.New("content").Parse(source))
return t
}
var rootTemplate = contentTemplate(`
<h1>Memsize</h1>
{{template "rootbuttons" .}}
<hr/>
<h3>Reports</h3>
<ul>
{{range .Reports}}
<li><a href="{{printf "%d" | $.Link "report/"}}">{{quote .RootName}} @ {{.Date}}</a></li>
{{else}}
No reports yet, hit a scan button to create one.
{{end}}
</ul>
`)
var notFoundTemplate = contentTemplate(`
<h1>{{.Data}}</h1>
{{template "rootbuttons" .}}
`)
var reportTemplate = contentTemplate(`
{{- $report := .Data -}}
<h1>Memsize Report {{$report.ID}}</h1>
<form method="POST" action="{{$.Link "scan?root=" $report.RootName}}">
<a class="button" href="{{$.Link ""}}">Overview</a>
<button type="submit">Scan Again</button>
</form>
<pre>
Root: {{quote $report.RootName}}
Date: {{$report.Date}}
Duration: {{$report.Duration}}
Bitmap Size: {{$report.Sizes.BitmapSize | humansize}}
Bitmap Utilization: {{$report.Sizes.BitmapUtilization}}
</pre>
<hr/>
<pre>
{{$report.Sizes.Report}}
</pre>
`)

153
vendor/github.com/fjl/memsize/memsizeui/ui.go generated vendored Normal file
View File

@@ -0,0 +1,153 @@
package memsizeui
import (
"bytes"
"fmt"
"html/template"
"net/http"
"reflect"
"sort"
"strings"
"sync"
"time"
"github.com/fjl/memsize"
)
type Handler struct {
init sync.Once
mux http.ServeMux
mu sync.Mutex
reports map[int]Report
roots map[string]interface{}
reportID int
}
type Report struct {
ID int
Date time.Time
Duration time.Duration
RootName string
Sizes memsize.Sizes
}
type templateInfo struct {
Roots []string
Reports map[int]Report
PathDepth int
Data interface{}
}
func (ti *templateInfo) Link(path ...string) string {
prefix := strings.Repeat("../", ti.PathDepth)
return prefix + strings.Join(path, "")
}
func (h *Handler) Add(name string, v interface{}) {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
panic("root must be non-nil pointer")
}
h.mu.Lock()
if h.roots == nil {
h.roots = make(map[string]interface{})
}
h.roots[name] = v
h.mu.Unlock()
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.init.Do(func() {
h.reports = make(map[int]Report)
h.mux.HandleFunc("/", h.handleRoot)
h.mux.HandleFunc("/scan", h.handleScan)
h.mux.HandleFunc("/report/", h.handleReport)
})
h.mux.ServeHTTP(w, r)
}
func (h *Handler) templateInfo(r *http.Request, data interface{}) *templateInfo {
h.mu.Lock()
roots := make([]string, 0, len(h.roots))
for name := range h.roots {
roots = append(roots, name)
}
h.mu.Unlock()
sort.Strings(roots)
return &templateInfo{
Roots: roots,
Reports: h.reports,
PathDepth: strings.Count(r.URL.Path, "/") - 1,
Data: data,
}
}
func (h *Handler) handleRoot(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
serveHTML(w, rootTemplate, http.StatusOK, h.templateInfo(r, nil))
}
func (h *Handler) handleScan(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "invalid HTTP method, want POST", http.StatusMethodNotAllowed)
return
}
ti := h.templateInfo(r, "Unknown root")
id, ok := h.scan(r.URL.Query().Get("root"))
if !ok {
serveHTML(w, notFoundTemplate, http.StatusNotFound, ti)
return
}
w.Header().Add("Location", ti.Link(fmt.Sprintf("report/%d", id)))
w.WriteHeader(http.StatusSeeOther)
}
func (h *Handler) handleReport(w http.ResponseWriter, r *http.Request) {
var id int
fmt.Sscan(strings.TrimPrefix(r.URL.Path, "/report/"), &id)
h.mu.Lock()
report, ok := h.reports[id]
h.mu.Unlock()
if !ok {
serveHTML(w, notFoundTemplate, http.StatusNotFound, h.templateInfo(r, "Report not found"))
} else {
serveHTML(w, reportTemplate, http.StatusOK, h.templateInfo(r, report))
}
}
func (h *Handler) scan(root string) (int, bool) {
h.mu.Lock()
defer h.mu.Unlock()
val, ok := h.roots[root]
if !ok {
return 0, false
}
id := h.reportID
start := time.Now()
sizes := memsize.Scan(val)
h.reports[id] = Report{
ID: id,
RootName: root,
Date: start.Truncate(1 * time.Second),
Duration: time.Since(start),
Sizes: sizes,
}
h.reportID++
return id, true
}
func serveHTML(w http.ResponseWriter, tpl *template.Template, status int, ti *templateInfo) {
w.Header().Set("content-type", "text/html")
var buf bytes.Buffer
if err := tpl.Execute(&buf, ti); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
buf.WriteTo(w)
}

14
vendor/github.com/fjl/memsize/runtimefunc.go generated vendored Normal file
View File

@@ -0,0 +1,14 @@
package memsize
import "unsafe"
var _ = unsafe.Pointer(nil)
//go:linkname stopTheWorld runtime.stopTheWorld
func stopTheWorld(reason string)
//go:linkname startTheWorld runtime.startTheWorld
func startTheWorld()
//go:linkname chanbuf runtime.chanbuf
func chanbuf(ch unsafe.Pointer, i uint) unsafe.Pointer

1
vendor/github.com/fjl/memsize/runtimefunc.s generated vendored Normal file
View File

@@ -0,0 +1 @@
// This file is required to make stub function declarations work.

119
vendor/github.com/fjl/memsize/type.go generated vendored Normal file
View File

@@ -0,0 +1,119 @@
package memsize
import (
"fmt"
"reflect"
)
// address is a memory location.
//
// Code dealing with uintptr is oblivious to the zero address.
// Code dealing with address is not: it treats the zero address
// as invalid. Offsetting an invalid address doesn't do anything.
//
// This distinction is useful because there are objects that we can't
// get the pointer to.
type address uintptr
const invalidAddr = address(0)
func (a address) valid() bool {
return a != 0
}
func (a address) addOffset(off uintptr) address {
if !a.valid() {
return invalidAddr
}
return a + address(off)
}
func (a address) String() string {
if uintptrBits == 32 {
return fmt.Sprintf("%#0.8x", uintptr(a))
}
return fmt.Sprintf("%#0.16x", uintptr(a))
}
type typCache map[reflect.Type]typInfo
type typInfo struct {
isPointer bool
needScan bool
}
// isPointer returns true for pointer-ish values. The notion of
// pointer includes everything but plain values, i.e. slices, maps
// channels, interfaces are 'pointer', too.
func (tc *typCache) isPointer(typ reflect.Type) bool {
return tc.info(typ).isPointer
}
// needScan reports whether a value of the type needs to be scanned
// recursively because it may contain pointers.
func (tc *typCache) needScan(typ reflect.Type) bool {
return tc.info(typ).needScan
}
func (tc *typCache) info(typ reflect.Type) typInfo {
info, found := (*tc)[typ]
switch {
case found:
return info
case isPointer(typ):
info = typInfo{true, true}
default:
info = typInfo{false, tc.checkNeedScan(typ)}
}
(*tc)[typ] = info
return info
}
func (tc *typCache) checkNeedScan(typ reflect.Type) bool {
switch k := typ.Kind(); k {
case reflect.Struct:
// Structs don't need scan if none of their fields need it.
for i := 0; i < typ.NumField(); i++ {
if tc.needScan(typ.Field(i).Type) {
return true
}
}
case reflect.Array:
// Arrays don't need scan if their element type doesn't.
return tc.needScan(typ.Elem())
}
return false
}
func isPointer(typ reflect.Type) bool {
k := typ.Kind()
switch {
case k <= reflect.Complex128:
return false
case k == reflect.Array:
return false
case k >= reflect.Chan && k <= reflect.String:
return true
case k == reflect.Struct || k == reflect.UnsafePointer:
return false
default:
unhandledKind(k)
return false
}
}
func unhandledKind(k reflect.Kind) {
panic("unhandled kind " + k.String())
}
// HumanSize formats the given number of bytes as a readable string.
func HumanSize(bytes uintptr) string {
switch {
case bytes < 1024:
return fmt.Sprintf("%d B", bytes)
case bytes < 1024*1024:
return fmt.Sprintf("%.3f KB", float64(bytes)/1024)
default:
return fmt.Sprintf("%.3f MB", float64(bytes)/1024/1024)
}
}

View File

@@ -12,7 +12,11 @@ import (
"sync"
)
const typeShift = 3
const typeShift = 4
// Verify at compile-time that typeShift is large enough to cover all FileType
// values by confirming that 0 == 0.
var _ [0]struct{} = [TypeAll >> typeShift]struct{}{}
type memStorageLock struct {
ms *memStorage
@@ -143,7 +147,7 @@ func (ms *memStorage) Remove(fd FileDesc) error {
}
func (ms *memStorage) Rename(oldfd, newfd FileDesc) error {
if FileDescOk(oldfd) || FileDescOk(newfd) {
if !FileDescOk(oldfd) || !FileDescOk(newfd) {
return ErrInvalidFile
}
if oldfd == newfd {

View File

@@ -20,7 +20,7 @@ func shorten(str string) string {
return str[:3] + ".." + str[len(str)-3:]
}
var bunits = [...]string{"", "Ki", "Mi", "Gi"}
var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"}
func shortenb(bytes int) string {
i := 0

64
vendor/vendor.json vendored
View File

@@ -110,6 +110,18 @@
"revision": "5ec5d9d3c2cf82e9688b34e9bc27a94d616a7193",
"revisionTime": "2017-02-09T08:00:14Z"
},
{
"checksumSHA1": "Jq1rrHSGPfh689nA2hL1QVb62zE=",
"path": "github.com/fjl/memsize",
"revision": "ca190fb6ffbc076ff49197b7168a760f30182d2e",
"revisionTime": "2018-04-18T12:24:29Z"
},
{
"checksumSHA1": "Z13QAYTqeW4cTiglkc2F05gWLu4=",
"path": "github.com/fjl/memsize/memsizeui",
"revision": "ca190fb6ffbc076ff49197b7168a760f30182d2e",
"revisionTime": "2018-04-18T12:24:29Z"
},
{
"checksumSHA1": "0orwvPL96wFckVJyPl39fz2QsgA=",
"path": "github.com/gizak/termui",
@@ -406,76 +418,76 @@
"revisionTime": "2017-07-05T02:17:15Z"
},
{
"checksumSHA1": "3QsnhPTXGytTbW3uDvQLgSo9s9M=",
"checksumSHA1": "k13cCuMJO7+KhR8ZXx5oUqDKGQA=",
"path": "github.com/syndtr/goleveldb/leveldb",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=",
"path": "github.com/syndtr/goleveldb/leveldb/cache",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=",
"path": "github.com/syndtr/goleveldb/leveldb/comparer",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=",
"path": "github.com/syndtr/goleveldb/leveldb/errors",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=",
"path": "github.com/syndtr/goleveldb/leveldb/filter",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=",
"path": "github.com/syndtr/goleveldb/leveldb/iterator",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=",
"path": "github.com/syndtr/goleveldb/leveldb/journal",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=",
"path": "github.com/syndtr/goleveldb/leveldb/memdb",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=",
"path": "github.com/syndtr/goleveldb/leveldb/opt",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "QCSae2ub87f8awH+PKMpd8ZYOtg=",
"checksumSHA1": "7H3fa12T7WoMAeXq1+qG5O7LD0w=",
"path": "github.com/syndtr/goleveldb/leveldb/storage",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=",
"path": "github.com/syndtr/goleveldb/leveldb/table",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=",
"path": "github.com/syndtr/goleveldb/leveldb/util",
"revision": "169b1b37be738edb2813dab48c97a549bcf99bb5",
"revisionTime": "2018-03-07T11:33:52Z"
"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
"revisionTime": "2018-05-02T07:23:49Z"
},
{
"checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=",

View File

@@ -67,7 +67,6 @@ func (sc *Client) SetMaxMessageSize(ctx context.Context, size uint32) error {
}
// SetMinimumPoW (experimental) sets the minimal PoW required by this node.
// This experimental function was introduced for the future dynamic adjustment of
// PoW requirement. If the node is overwhelmed with messages, it should raise the
// PoW requirement and notify the peers. The new value should be set relative to
@@ -77,7 +76,7 @@ func (sc *Client) SetMinimumPoW(ctx context.Context, pow float64) error {
return sc.c.CallContext(ctx, &ignored, "shh_setMinPoW", pow)
}
// Marks specific peer trusted, which will allow it to send historic (expired) messages.
// MarkTrustedPeer marks specific peer trusted, which will allow it to send historic (expired) messages.
// Note This function is not adding new nodes, the node needs to exists as a peer.
func (sc *Client) MarkTrustedPeer(ctx context.Context, enode string) error {
var ignored bool

View File

@@ -89,7 +89,7 @@ func (api *PublicWhisperAPI) SetMaxMessageSize(ctx context.Context, size uint32)
return true, api.w.SetMaxMessageSize(size)
}
// SetMinPow sets the minimum PoW for a message before it is accepted.
// SetMinPoW sets the minimum PoW for a message before it is accepted.
func (api *PublicWhisperAPI) SetMinPoW(ctx context.Context, pow float64) (bool, error) {
return true, api.w.SetMinimumPoW(pow)
}
@@ -142,7 +142,7 @@ func (api *PublicWhisperAPI) GetPublicKey(ctx context.Context, id string) (hexut
return crypto.FromECDSAPub(&key.PublicKey), nil
}
// GetPublicKey returns the private key associated with the given key. The key is the hex
// GetPrivateKey returns the private key associated with the given key. The key is the hex
// encoded representation of a key in the form specified in section 4.3.6 of ANSI X9.62.
func (api *PublicWhisperAPI) GetPrivateKey(ctx context.Context, id string) (hexutil.Bytes, error) {
key, err := api.w.GetPrivateKey(id)

View File

@@ -15,7 +15,7 @@
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
Package whisper implements the Whisper protocol (version 5).
Package whisperv5 implements the Whisper protocol (version 5).
Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP).
As such it may be likened and compared to both, not dissimilar to the

View File

@@ -33,7 +33,7 @@ import (
"github.com/ethereum/go-ethereum/log"
)
// Options specifies the exact way a message should be wrapped into an Envelope.
// MessageParams specifies the exact way a message should be wrapped into an Envelope.
type MessageParams struct {
TTL uint32
Src *ecdsa.PrivateKey
@@ -86,7 +86,7 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool {
return msg.Dst != nil
}
// NewMessage creates and initializes a non-signed, non-encrypted Whisper message.
// NewSentMessage creates and initializes a non-signed, non-encrypted Whisper message.
func NewSentMessage(params *MessageParams) (*sentMessage, error) {
msg := sentMessage{}
msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit)
@@ -330,7 +330,7 @@ func (msg *ReceivedMessage) extractPadding(end int) (int, bool) {
return paddingSize, true
}
// Recover retrieves the public key of the message signer.
// SigToPubKey retrieves the public key of the message signer.
func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey {
defer func() { recover() }() // in case of invalid signature

View File

@@ -27,7 +27,7 @@ import (
set "gopkg.in/fatih/set.v0"
)
// peer represents a whisper protocol peer connection.
// Peer represents a whisper protocol peer connection.
type Peer struct {
host *Whisper
peer *p2p.Peer
@@ -53,51 +53,51 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer {
// start initiates the peer updater, periodically broadcasting the whisper packets
// into the network.
func (p *Peer) start() {
go p.update()
log.Trace("start", "peer", p.ID())
func (peer *Peer) start() {
go peer.update()
log.Trace("start", "peer", peer.ID())
}
// stop terminates the peer updater, stopping message forwarding to it.
func (p *Peer) stop() {
close(p.quit)
log.Trace("stop", "peer", p.ID())
func (peer *Peer) stop() {
close(peer.quit)
log.Trace("stop", "peer", peer.ID())
}
// handshake sends the protocol initiation status message to the remote peer and
// verifies the remote status too.
func (p *Peer) handshake() error {
func (peer *Peer) handshake() error {
// Send the handshake status message asynchronously
errc := make(chan error, 1)
go func() {
errc <- p2p.Send(p.ws, statusCode, ProtocolVersion)
errc <- p2p.Send(peer.ws, statusCode, ProtocolVersion)
}()
// Fetch the remote status packet and verify protocol match
packet, err := p.ws.ReadMsg()
packet, err := peer.ws.ReadMsg()
if err != nil {
return err
}
if packet.Code != statusCode {
return fmt.Errorf("peer [%x] sent packet %x before status packet", p.ID(), packet.Code)
return fmt.Errorf("peer [%x] sent packet %x before status packet", peer.ID(), packet.Code)
}
s := rlp.NewStream(packet.Payload, uint64(packet.Size))
peerVersion, err := s.Uint()
if err != nil {
return fmt.Errorf("peer [%x] sent bad status message: %v", p.ID(), err)
return fmt.Errorf("peer [%x] sent bad status message: %v", peer.ID(), err)
}
if peerVersion != ProtocolVersion {
return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", p.ID(), peerVersion, ProtocolVersion)
return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", peer.ID(), peerVersion, ProtocolVersion)
}
// Wait until out own status is consumed too
if err := <-errc; err != nil {
return fmt.Errorf("peer [%x] failed to send status packet: %v", p.ID(), err)
return fmt.Errorf("peer [%x] failed to send status packet: %v", peer.ID(), err)
}
return nil
}
// update executes periodic operations on the peer, including message transmission
// and expiration.
func (p *Peer) update() {
func (peer *Peer) update() {
// Start the tickers for the updates
expire := time.NewTicker(expirationCycle)
transmit := time.NewTicker(transmissionCycle)
@@ -106,15 +106,15 @@ func (p *Peer) update() {
for {
select {
case <-expire.C:
p.expire()
peer.expire()
case <-transmit.C:
if err := p.broadcast(); err != nil {
log.Trace("broadcast failed", "reason", err, "peer", p.ID())
if err := peer.broadcast(); err != nil {
log.Trace("broadcast failed", "reason", err, "peer", peer.ID())
return
}
case <-p.quit:
case <-peer.quit:
return
}
}
@@ -148,16 +148,16 @@ func (peer *Peer) expire() {
// broadcast iterates over the collection of envelopes and transmits yet unknown
// ones over the network.
func (p *Peer) broadcast() error {
func (peer *Peer) broadcast() error {
var cnt int
envelopes := p.host.Envelopes()
envelopes := peer.host.Envelopes()
for _, envelope := range envelopes {
if !p.marked(envelope) {
err := p2p.Send(p.ws, messagesCode, envelope)
if !peer.marked(envelope) {
err := p2p.Send(peer.ws, messagesCode, envelope)
if err != nil {
return err
} else {
p.mark(envelope)
peer.mark(envelope)
cnt++
}
}
@@ -168,7 +168,7 @@ func (p *Peer) broadcast() error {
return nil
}
func (p *Peer) ID() []byte {
id := p.peer.ID()
func (peer *Peer) ID() []byte {
id := peer.peer.ID()
return id[:]
}

View File

@@ -32,7 +32,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/nat"
)
var keys []string = []string{
var keys = []string{
"d49dcf37238dc8a7aac57dc61b9fee68f0a97f062968978b9fafa7d1033d03a9",
"73fd6143c48e80ed3c56ea159fe7494a0b6b393a392227b422f4c3e8f1b54f98",
"119dd32adb1daa7a4c7bf77f847fb28730785aa92947edf42fdd997b54de40dc",
@@ -84,9 +84,9 @@ type TestNode struct {
var result TestData
var nodes [NumNodes]*TestNode
var sharedKey []byte = []byte("some arbitrary data here")
var sharedKey = []byte("some arbitrary data here")
var sharedTopic TopicType = TopicType{0xF, 0x1, 0x2, 0}
var expectedMessage []byte = []byte("per rectum ad astra")
var expectedMessage = []byte("per rectum ad astra")
// This test does the following:
// 1. creates a chain of whisper nodes,

View File

@@ -23,7 +23,7 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
)
// Topic represents a cryptographically secure, probabilistic partial
// TopicType represents a cryptographically secure, probabilistic partial
// classifications of a message, determined as the first (left) 4 bytes of the
// SHA3 hash of some arbitrary data given by the original author of the message.
type TopicType [TopicLength]byte

View File

@@ -469,18 +469,18 @@ func (w *Whisper) Stop() error {
// HandlePeer is called by the underlying P2P layer when the whisper sub-protocol
// connection is negotiated.
func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
func (w *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
// Create the new peer and start tracking it
whisperPeer := newPeer(wh, peer, rw)
whisperPeer := newPeer(w, peer, rw)
wh.peerMu.Lock()
wh.peers[whisperPeer] = struct{}{}
wh.peerMu.Unlock()
w.peerMu.Lock()
w.peers[whisperPeer] = struct{}{}
w.peerMu.Unlock()
defer func() {
wh.peerMu.Lock()
delete(wh.peers, whisperPeer)
wh.peerMu.Unlock()
w.peerMu.Lock()
delete(w.peers, whisperPeer)
w.peerMu.Unlock()
}()
// Run the peer handshake and state updates
@@ -490,11 +490,11 @@ func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
whisperPeer.start()
defer whisperPeer.stop()
return wh.runMessageLoop(whisperPeer, rw)
return w.runMessageLoop(whisperPeer, rw)
}
// runMessageLoop reads and processes inbound messages directly to merge into client-global state.
func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
func (w *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
for {
// fetch the next packet
packet, err := rw.ReadMsg()
@@ -502,7 +502,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
log.Warn("message loop", "peer", p.peer.ID(), "err", err)
return err
}
if packet.Size > wh.MaxMessageSize() {
if packet.Size > w.MaxMessageSize() {
log.Warn("oversized message received", "peer", p.peer.ID())
return errors.New("oversized message received")
}
@@ -518,7 +518,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
log.Warn("failed to decode envelope, peer will be disconnected", "peer", p.peer.ID(), "err", err)
return errors.New("invalid envelope")
}
cached, err := wh.add(&envelope)
cached, err := w.add(&envelope)
if err != nil {
log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err)
return errors.New("invalid envelope")
@@ -537,17 +537,17 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
log.Warn("failed to decode direct message, peer will be disconnected", "peer", p.peer.ID(), "err", err)
return errors.New("invalid direct message")
}
wh.postEvent(&envelope, true)
w.postEvent(&envelope, true)
}
case p2pRequestCode:
// Must be processed if mail server is implemented. Otherwise ignore.
if wh.mailServer != nil {
if w.mailServer != nil {
var request Envelope
if err := packet.Decode(&request); err != nil {
log.Warn("failed to decode p2p request message, peer will be disconnected", "peer", p.peer.ID(), "err", err)
return errors.New("invalid p2p request")
}
wh.mailServer.DeliverMail(p, &request)
w.mailServer.DeliverMail(p, &request)
}
default:
// New message types might be implemented in the future versions of Whisper.
@@ -561,29 +561,27 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
// add inserts a new envelope into the message pool to be distributed within the
// whisper network. It also inserts the envelope into the expiration pool at the
// appropriate time-stamp. In case of error, connection should be dropped.
func (wh *Whisper) add(envelope *Envelope) (bool, error) {
func (w *Whisper) add(envelope *Envelope) (bool, error) {
now := uint32(time.Now().Unix())
sent := envelope.Expiry - envelope.TTL
if sent > now {
if sent-SynchAllowance > now {
return false, fmt.Errorf("envelope created in the future [%x]", envelope.Hash())
} else {
// recalculate PoW, adjusted for the time difference, plus one second for latency
envelope.calculatePoW(sent - now + 1)
}
// recalculate PoW, adjusted for the time difference, plus one second for latency
envelope.calculatePoW(sent - now + 1)
}
if envelope.Expiry < now {
if envelope.Expiry+SynchAllowance*2 < now {
return false, fmt.Errorf("very old message")
} else {
log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex())
return false, nil // drop envelope without error
}
log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex())
return false, nil // drop envelope without error
}
if uint32(envelope.size()) > wh.MaxMessageSize() {
if uint32(envelope.size()) > w.MaxMessageSize() {
return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash())
}
@@ -598,36 +596,36 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash())
}
if envelope.PoW() < wh.MinPow() {
if envelope.PoW() < w.MinPow() {
log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex())
return false, nil // drop envelope without error
}
hash := envelope.Hash()
wh.poolMu.Lock()
_, alreadyCached := wh.envelopes[hash]
w.poolMu.Lock()
_, alreadyCached := w.envelopes[hash]
if !alreadyCached {
wh.envelopes[hash] = envelope
if wh.expirations[envelope.Expiry] == nil {
wh.expirations[envelope.Expiry] = set.NewNonTS()
w.envelopes[hash] = envelope
if w.expirations[envelope.Expiry] == nil {
w.expirations[envelope.Expiry] = set.NewNonTS()
}
if !wh.expirations[envelope.Expiry].Has(hash) {
wh.expirations[envelope.Expiry].Add(hash)
if !w.expirations[envelope.Expiry].Has(hash) {
w.expirations[envelope.Expiry].Add(hash)
}
}
wh.poolMu.Unlock()
w.poolMu.Unlock()
if alreadyCached {
log.Trace("whisper envelope already cached", "hash", envelope.Hash().Hex())
} else {
log.Trace("cached whisper envelope", "hash", envelope.Hash().Hex())
wh.statsMu.Lock()
wh.stats.memoryUsed += envelope.size()
wh.statsMu.Unlock()
wh.postEvent(envelope, false) // notify the local node about the new message
if wh.mailServer != nil {
wh.mailServer.Archive(envelope)
w.statsMu.Lock()
w.stats.memoryUsed += envelope.size()
w.statsMu.Unlock()
w.postEvent(envelope, false) // notify the local node about the new message
if w.mailServer != nil {
w.mailServer.Archive(envelope)
}
}
return true, nil
@@ -838,9 +836,8 @@ func deriveKeyMaterial(key []byte, version uint64) (derivedKey []byte, err error
// because it's a once in a session experience
derivedKey := pbkdf2.Key(key, nil, 65356, aesKeyLength, sha256.New)
return derivedKey, nil
} else {
return nil, unknownVersionError(version)
}
return nil, unknownVersionError(version)
}
// GenerateRandomID generates a random string, which is then returned to be used as a key id