diff --git a/.gitignore b/.gitignore index 5ded4fca..626f0ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ rosetta-bitcoin bitcoin-data cli-data +.DS_Store +rosetta-sdk-go \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..b13b91d5 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,95 @@ +# Copyright 2020 Coinbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build bitcoind +FROM ubuntu:18.04 as bitcoind-builder + +RUN mkdir -p /app \ + && chown -R nobody:nogroup /app +WORKDIR /app + +# Copy the local sdk to your container +COPY ./rosetta-sdk-go/* /app/ + +# Source: https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md#ubuntu--debian +RUN apt-get update && apt-get install -y make gcc g++ autoconf autotools-dev bsdmainutils build-essential git libboost-all-dev \ + libcurl4-openssl-dev libdb++-dev libevent-dev libssl-dev libtool pkg-config python python-pip libzmq3-dev wget + +# VERSION: Bitcoin Core 0.20.1 +RUN git clone https://github.com/bitcoin/bitcoin \ + && cd bitcoin \ + && git checkout 7ff64311bee570874c4f0dfa18f518552188df08 + +RUN cd bitcoin \ + && ./autogen.sh \ + && ./configure --disable-tests --without-miniupnpc --without-gui --with-incompatible-bdb --disable-hardening --disable-zmq --disable-bench --disable-wallet \ + && make + +RUN mv bitcoin/src/bitcoind /app/bitcoind \ + && rm -rf bitcoin + +# Build Rosetta Server Components +FROM ubuntu:18.04 as rosetta-builder + +RUN mkdir -p /app \ + && chown -R nobody:nogroup /app +WORKDIR /app + +RUN apt-get update && apt-get install -y curl make gcc g++ +ENV GOLANG_VERSION 1.15.5 +ENV GOLANG_DOWNLOAD_SHA256 9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz + +RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ + && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ + && tar -C /usr/local -xzf golang.tar.gz \ + && rm golang.tar.gz + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +# Use native remote build context to build in any directory +COPY . src +RUN cd src \ + && go build \ + && cd .. \ + && mv src/rosetta-bitcoin /app/rosetta-bitcoin \ + && mv src/assets/* /app \ + && rm -rf src + +## Build Final Image +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y libevent-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN mkdir -p /app \ + && chown -R nobody:nogroup /app \ + && mkdir -p /data \ + && chown -R nobody:nogroup /data + +WORKDIR /app + +# Copy binary from bitcoind-builder +COPY --from=bitcoind-builder /app/bitcoind /app/bitcoind + +# Copy binary from rosetta-builder +COPY --from=rosetta-builder /app/* /app/ + +# Set permissions for everything added to /app +RUN chmod -R 755 /app/* + +CMD ["/app/rosetta-bitcoin"] diff --git a/Makefile b/Makefile index 0737d29d..2f4ad9d5 100644 --- a/Makefile +++ b/Makefile @@ -25,22 +25,25 @@ build: build-local: docker build -t rosetta-bitcoin:latest . +build-local-dev: + docker build -f Dockerfile.dev -t rosetta-bitcoin:latest . + build-release: # make sure to always set version with vX.X.X docker build -t rosetta-bitcoin:$(version) .; docker save rosetta-bitcoin:$(version) | gzip > rosetta-bitcoin-$(version).tar.gz; run-mainnet-online: - docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest + docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest run-mainnet-offline: - docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest + docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest run-testnet-online: - docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest + docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest run-testnet-offline: - docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest + docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest train: ./zstd-train.sh $(network) transaction $(data-directory) diff --git a/README.md b/README.md index 3c71dfed..a8559311 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,16 @@ After cloning this repository, run: ```text make build-local ``` +#### From Source and Developing Locally +After cloning this repository, run: +```text +make build-local-dev +``` +Note: The purpose of this command is to build rosetta-bitcoin using a local sdk - this means you must have a local copy of rosetta-sdk-go in the same folder as the Dockefile and you must add the following to the bottom of your go.mod +```text +replace github.com/coinbase/rosetta-sdk-go v0.6.5 => ./rosetta-sdk-go +``` ### Run Running the following commands will start a Docker container in [detached mode](https://docs.docker.com/engine/reference/run/#detached--d) with @@ -61,25 +70,25 @@ at port `8080`. #### Mainnet:Online ```text -docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest +docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest ``` _If you cloned the repository, you can run `make run-mainnet-online`._ #### Mainnet:Offline ```text -docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest +docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest ``` _If you cloned the repository, you can run `make run-mainnet-offline`._ #### Testnet:Online ```text -docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest +docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest ``` _If you cloned the repository, you can run `make run-testnet-online`._ #### Testnet:Offline ```text -docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest +docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest ``` _If you cloned the repository, you can run `make run-testnet-offline`._ diff --git a/configuration/configuration.go b/configuration/configuration.go index 1c0c46eb..6d6fd7c2 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -17,6 +17,7 @@ package configuration import ( "errors" "fmt" + "github.com/coinbase/rosetta-sdk-go/syncer" "os" "path" "strconv" @@ -90,6 +91,10 @@ const ( // to determine mode. ModeEnv = "MODE" + // MaxSyncConcurrency is an environment variable + // used to cap the syncer concurrency + MaxSyncConcurrency = "MAXSYNC" + // NetworkEnv is the environment variable // read to determine network. NetworkEnv = "NETWORK" @@ -122,6 +127,7 @@ type Configuration struct { IndexerPath string BitcoindPath string Compressors []*encoder.CompressorEntry + MaxSyncConcurrency int64 } // LoadConfiguration attempts to create a new Configuration @@ -134,6 +140,26 @@ func LoadConfiguration(baseDirectory string) (*Configuration, error) { MinHeight: minPruneHeight, } + defaultMaxSync := false + maxSyncValue := os.Getenv(MaxSyncConcurrency) + if len(maxSyncValue) == 0 { + defaultMaxSync = true + } + + parsedValue, err := strconv.ParseInt(maxSyncValue, 10, 64) + if !(err == nil || defaultMaxSync) { + return nil, fmt.Errorf("%w: unable to parse maxsync %s", err, maxSyncValue) + } + + switch { + case defaultMaxSync: + config.MaxSyncConcurrency = syncer.DefaultMaxConcurrency + case parsedValue <= 0: + return nil, errors.New("syncer concurrency must be greater than zero") + default: + config.MaxSyncConcurrency = parsedValue + } + modeValue := Mode(os.Getenv(ModeEnv)) switch modeValue { case Online: diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 2b63bccb..dc10de78 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -30,9 +30,10 @@ import ( func TestLoadConfiguration(t *testing.T) { tests := map[string]struct { - Mode string - Network string - Port string + Mode string + Network string + Port string + MaxSyncConcurrency string cfg *Configuration err error @@ -44,15 +45,17 @@ func TestLoadConfiguration(t *testing.T) { Mode: string(Online), err: errors.New("NETWORK must be populated"), }, - "only mode and network set": { - Mode: string(Online), - Network: Mainnet, - err: errors.New("PORT must be populated"), + "port not set": { + Mode: string(Online), + Network: Mainnet, + MaxSyncConcurrency: "256", + err: errors.New("PORT must be populated"), }, "all set (mainnet)": { - Mode: string(Online), - Network: Mainnet, - Port: "1000", + Mode: string(Online), + Network: Mainnet, + Port: "1000", + MaxSyncConcurrency: "256", cfg: &Configuration{ Mode: Online, Network: &types.NetworkIdentifier{ @@ -63,6 +66,7 @@ func TestLoadConfiguration(t *testing.T) { Currency: bitcoin.MainnetCurrency, GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier, Port: 1000, + MaxSyncConcurrency: 256, RPCPort: mainnetRPCPort, ConfigPath: mainnetConfigPath, Pruning: &PruningConfiguration{ @@ -79,9 +83,41 @@ func TestLoadConfiguration(t *testing.T) { }, }, "all set (testnet)": { - Mode: string(Online), - Network: Testnet, - Port: "1000", + Mode: string(Online), + Network: Testnet, + Port: "1000", + MaxSyncConcurrency: "256", + cfg: &Configuration{ + Mode: Online, + Network: &types.NetworkIdentifier{ + Network: bitcoin.TestnetNetwork, + Blockchain: bitcoin.Blockchain, + }, + Params: bitcoin.TestnetParams, + Currency: bitcoin.TestnetCurrency, + GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier, + Port: 1000, + MaxSyncConcurrency: 256, + RPCPort: testnetRPCPort, + ConfigPath: testnetConfigPath, + Pruning: &PruningConfiguration{ + Frequency: pruneFrequency, + Depth: pruneDepth, + MinHeight: minPruneHeight, + }, + Compressors: []*encoder.CompressorEntry{ + { + Namespace: transactionNamespace, + DictionaryPath: testnetTransactionDictionary, + }, + }, + }, + }, + "default max sync set": { + Mode: string(Online), + Network: Testnet, + Port: "1000", + MaxSyncConcurrency: "", cfg: &Configuration{ Mode: Online, Network: &types.NetworkIdentifier{ @@ -92,6 +128,7 @@ func TestLoadConfiguration(t *testing.T) { Currency: bitcoin.TestnetCurrency, GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier, Port: 1000, + MaxSyncConcurrency: 256, RPCPort: testnetRPCPort, ConfigPath: testnetConfigPath, Pruning: &PruningConfiguration{ @@ -107,6 +144,20 @@ func TestLoadConfiguration(t *testing.T) { }, }, }, + "maxsync negative mainnet": { + Mode: string(Online), + Network: Mainnet, + Port: "1000", + MaxSyncConcurrency: "-2", + err: errors.New("syncer concurrency must be greater than zero"), + }, + "maxsync zero testnet": { + Mode: string(Online), + Network: Testnet, + Port: "1000", + MaxSyncConcurrency: "0", + err: errors.New("syncer concurrency must be greater than zero"), + }, "invalid mode": { Mode: "bad mode", Network: Testnet, @@ -136,6 +187,7 @@ func TestLoadConfiguration(t *testing.T) { os.Setenv(ModeEnv, test.Mode) os.Setenv(NetworkEnv, test.Network) os.Setenv(PortEnv, test.Port) + os.Setenv(MaxSyncConcurrency, test.MaxSyncConcurrency) cfg, err := LoadConfiguration(newDir) if test.err != nil { diff --git a/go.mod b/go.mod index 3d10bf54..1a8ae6fe 100644 --- a/go.mod +++ b/go.mod @@ -13,4 +13,4 @@ require ( golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect honnef.co/go/tools v0.0.1-2020.1.5 // indirect -) +) \ No newline at end of file diff --git a/go.sum b/go.sum index 75c15686..01926d8f 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= @@ -103,6 +104,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.24 h1:6AK+ORt3EMDO+FTjzXy/AQwHMbu52J2nYHIjyQX9azQ= github.com/ethereum/go-ethereum v1.9.24/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= +github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4IKNTfR0= +github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= @@ -138,6 +141,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= +github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -166,9 +171,11 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -276,6 +283,8 @@ github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.4 h1:JKsCsJqRVFz8eYCsQ5E/ANRbK6CanAtA9IUvGsXklyo= +github.com/tidwall/gjson v1.6.4/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= @@ -310,6 +319,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/indexer/indexer.go b/indexer/indexer.go index 9f8bb199..4f9f3e97 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -123,6 +123,8 @@ type Indexer struct { seenMutex sync.Mutex seenSemaphore *semaphore.Weighted + + maxSync int64 } // CloseDatabase closes a storage.Database. This should be called @@ -229,6 +231,7 @@ func Initialize( coinCache: map[string]*types.AccountCoin{}, coinCacheMutex: new(sdkUtils.PriorityMutex), seenSemaphore: semaphore.NewWeighted(int64(runtime.NumCPU())), + maxSync: config.MaxSyncConcurrency, } coinStorage := modules.NewCoinStorage( @@ -296,6 +299,7 @@ func (i *Indexer) Sync(ctx context.Context) error { syncer.WithCacheSize(syncer.DefaultCacheSize), syncer.WithSizeMultiplier(sizeMultiplier), syncer.WithPastBlocks(pastBlocks), + syncer.WithMaxConcurrency(i.maxSync), ) return syncer.Sync(ctx, startIndex, indexPlaceholder) diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index 08e05d31..99ceb9bc 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -62,6 +62,7 @@ func TestIndexer_Pruning(t *testing.T) { Network: bitcoin.MainnetNetwork, Blockchain: bitcoin.Blockchain, }, + MaxSyncConcurrency: 256, GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier, Pruning: &configuration.PruningConfiguration{ Frequency: 50 * time.Millisecond, @@ -227,6 +228,7 @@ func TestIndexer_Transactions(t *testing.T) { Network: bitcoin.MainnetNetwork, Blockchain: bitcoin.Blockchain, }, + MaxSyncConcurrency: 256, GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier, IndexerPath: newDir, } @@ -445,6 +447,7 @@ func TestIndexer_Reorg(t *testing.T) { Network: bitcoin.MainnetNetwork, Blockchain: bitcoin.Blockchain, }, + MaxSyncConcurrency: 256, GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier, IndexerPath: newDir, } @@ -687,6 +690,7 @@ func TestIndexer_HeaderReorg(t *testing.T) { Network: bitcoin.MainnetNetwork, Blockchain: bitcoin.Blockchain, }, + MaxSyncConcurrency: 256, GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier, IndexerPath: newDir, }