catch up

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*.md]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = false
+max_line_length = 80
+insert_final_newline = true
+charset = utf-8
+end_of_line = lf

.gitlab-ci.yml

@@ -157,15 +157,4 @@ backend-deploy-and-test-zcash_testnet:
       - configs/coins/zcash_testnet.json
   tags:
     - blockbook
-  script: ./contrib/scripts/backend-deploy-and-test.sh zcash_testnet zcash-testnet zcash=test testnet3/debug.log
-
-backend-deploy-and-test-goerli-archive:
-  stage: backend-deploy-and-test
-  only:
-    refs:
-      - master
-    changes:
-      - configs/coins/ethereum_testnet_goerli_archive.json
-  tags:
-    - blockbook
-  script: ./contrib/scripts/backend-deploy-and-test.sh ethereum_testnet_goerli_archive ethereum-testnet-goerli-archive ethereum=test ethereum_testnet_goerli_archive.log
+  script: ./contrib/scripts/backend-deploy-and-test.sh zcash_testnet zcash-testnet zcash=test testnet3/debug.log

.prettierrc

@@ -0,0 +1,11 @@
+{
+    "printWidth": 100,
+    "arrowParens": "avoid",
+    "bracketSpacing": true,
+    "singleQuote": true,
+    "semi": true,
+    "trailingComma": "all",
+    "tabWidth": 4,
+    "useTabs": false,
+    "bracketSameLine": false
+}

CONTRIBUTING.md

@@ -75,7 +75,7 @@ also in [build guide](/docs/build.md#on-naming-conventions-and-versioning). You
 *mainnet* option.
 
 In the section *blockbook* update information how to build and configure Blockbook service. Usually they are only
-*package_name*, *system_user* and *explorer_url* options. Naming conventions are are described
+*package_name*, *system_user* and *explorer_url* options. Naming conventions are described
 [here](/docs/build.md#on-naming-conventions-and-versioning).
 
 Update *package_maintainer* and *package_maintainer_email* options in the section *meta*.

Makefile

@@ -1,9 +1,10 @@
 BIN_IMAGE = blockbook-build
 DEB_IMAGE = blockbook-build-deb
 PACKAGER = $(shell id -u):$(shell id -g)
+DOCKER_VERSION = $(shell docker version --format '{{.Client.Version}}')
 BASE_IMAGE = $$(awk -F= '$$1=="ID" { print $$2 ;}' /etc/os-release):$$(awk -F= '$$1=="VERSION_ID" { print $$2 ;}' /etc/os-release | tr -d '"')
 NO_CACHE = false
-TCMALLOC = 
+TCMALLOC =
 PORTABLE = 0
 ARGS ?=
 
@@ -27,13 +28,13 @@ test-all: .bin-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
 
 deb-backend-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
 
 deb-blockbook-%: .deb-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
 
 deb-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
 
 deb-blockbook-all: clean-deb $(addprefix deb-blockbook-, $(TARGETS))
 
@@ -55,7 +56,7 @@ build-images: clean-images
 .deb-image: .bin-image
 	@if [ $$(build/tools/image_status.sh $(DEB_IMAGE):latest build/docker) != "ok" ]; then \
 		echo "Building image $(DEB_IMAGE)..."; \
-		docker build --no-cache=$(NO_CACHE) -t $(DEB_IMAGE) build/docker/deb; \
+		docker build --no-cache=$(NO_CACHE) --build-arg DOCKER_VERSION=$(DOCKER_VERSION) -t $(DEB_IMAGE) build/docker/deb; \
 	else \
 		echo "Image $(DEB_IMAGE) is up to date"; \
 	fi
@@ -79,3 +80,6 @@ clean-bin-image:
 
 clean-deb-image:
 	- docker rmi $(DEB_IMAGE)
+
+style:
+	find . -name "*.go" -exec gofmt -w {} \;

README.md

@@ -2,14 +2,14 @@
 
 # Blockbook
 
-**Blockbook** is back-end service for Trezor wallet. Main features of **Blockbook** are:
+**Blockbook** is a back-end service for Trezor Suite. The main features of **Blockbook** are:
 
-- index of addresses and address balances of the connected block chain
-- fast index search
-- simple blockchain explorer
-- websocket, API and legacy Bitcore Insight compatible socket.io interfaces
-- support of multiple coins (Bitcoin and Ethereum type) with easy extensibility to other coins
-- scripts for easy creation of debian packages for backend and blockbook
+-   index of addresses and address balances of the connected block chain
+-   fast index search
+-   simple blockchain explorer
+-   websocket, API and legacy Bitcore Insight compatible socket.io interfaces
+-   support of multiple coins (Bitcoin and Ethereum type) with easy extensibility to other coins
+-   scripts for easy creation of debian packages for backend and blockbook
 
 ## Build and installation instructions
 
@@ -19,42 +19,43 @@ Memory and disk requirements for initial synchronization of **Bitcoin mainnet**
 Other coins should have lower requirements, depending on the size of their block chain. Note that fast SSD disks are highly
 recommended.
 
-User installation guide is [here](https://wiki.trezor.io/User_manual:Running_a_local_instance_of_Trezor_Wallet_backend_(Blockbook)).
+User installation guide is [here](<https://wiki.trezor.io/User_manual:Running_a_local_instance_of_Trezor_Wallet_backend_(Blockbook)>).
 
 Developer build guide is [here](/docs/build.md).
 
 Contribution guide is [here](CONTRIBUTING.md).
 
 ## Implemented coins
 
-Blockbook currently supports over 30 coins. The Trezor team implemented 
+Blockbook currently supports over 30 coins. The Trezor team implemented
 
-- Bitcoin, Bitcoin Cash, Zcash, Dash, Litecoin, Bitcoin Gold, Ethereum, Ethereum Classic, Dogecoin, Namecoin, Vertcoin, DigiByte, Liquid
+-   Bitcoin, Bitcoin Cash, Zcash, Dash, Litecoin, Bitcoin Gold, Ethereum, Ethereum Classic, Dogecoin, Namecoin, Vertcoin, DigiByte, Liquid
 
 the rest of coins were implemented by the community.
 
 Testnets for some coins are also supported, for example:
-- Bitcoin Testnet, Bitcoin Cash Testnet, ZCash Testnet, Ethereum Testnet Ropsten
+
+-   Bitcoin Testnet, Bitcoin Cash Testnet, ZCash Testnet, Ethereum Testnets (Sepolia, Hoodi)
 
 List of all implemented coins is in [the registry of ports](/docs/ports.md).
 
 ## Common issues when running Blockbook or implementing additional coins
 
 #### Out of memory when doing initial synchronization
 
-How to reduce memory footprint of the initial sync: 
+How to reduce memory footprint of the initial sync:
 
-- disable rocksdb cache by parameter `-dbcache=0`, the default size is 500MB
-- run blockbook with parameter `-workers=1`. This disables bulk import mode, which caches a lot of data in memory (not in rocksdb cache). It will run about twice as slowly but especially for smaller blockchains it is no problem at all.
+-   disable rocksdb cache by parameter `-dbcache=0`, the default size is 500MB
+-   run blockbook with parameter `-workers=1`. This disables bulk import mode, which caches a lot of data in memory (not in rocksdb cache). It will run about twice as slowly but especially for smaller blockchains it is no problem at all.
 
 Please add your experience to this [issue](https://github.com/trezor/blockbook/issues/43).
 
 #### Error `internalState: database is in inconsistent state and cannot be used`
 
-Blockbook was killed during the initial import, most commonly by OOM killer. 
-By default, Blockbook performs the initial import in bulk import mode, which for performance reasons does not store all data immediately to the database. If Blockbook is killed during this phase, the database is left in an inconsistent state. 
+Blockbook was killed during the initial import, most commonly by OOM killer.
+By default, Blockbook performs the initial import in bulk import mode, which for performance reasons does not store all data immediately to the database. If Blockbook is killed during this phase, the database is left in an inconsistent state.
 
-See above how to reduce the memory footprint, delete the database files and run the import again. 
+See above how to reduce the memory footprint, delete the database files and run the import again.
 
 Check [this](https://github.com/trezor/blockbook/issues/89) or [this](https://github.com/trezor/blockbook/issues/147) issue for more info.
 
@@ -73,3 +74,7 @@ Blockbook stores data the key-value store RocksDB. Database format is described
 ## API
 
 Blockbook API is described [here](/docs/api.md).
+
+## Environment variables
+
+List of environment variables that affect Blockbook's behavior is [here](/docs/env.md).

api/ethereumtype.go

@@ -0,0 +1,92 @@
+package api
+
+import (
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/linxGnu/grocksdb"
+	"github.com/trezor/blockbook/bchain"
+	"github.com/trezor/blockbook/db"
+)
+
+// refetch internal data
+var refetchingInternalData = false
+var refetchInternalDataMux sync.Mutex
+
+func (w *Worker) IsRefetchingInternalData() bool {
+	refetchInternalDataMux.Lock()
+	defer refetchInternalDataMux.Unlock()
+	return refetchingInternalData
+}
+
+func (w *Worker) RefetchInternalData() error {
+	refetchInternalDataMux.Lock()
+	defer refetchInternalDataMux.Unlock()
+	if !refetchingInternalData {
+		refetchingInternalData = true
+		go w.RefetchInternalDataRoutine()
+	}
+	return nil
+}
+
+const maxNumberOfRetires = 25
+
+func (w *Worker) incrementRefetchInternalDataRetryCount(ie *db.BlockInternalDataError) {
+	wb := grocksdb.NewWriteBatch()
+	defer wb.Destroy()
+	err := w.db.StoreBlockInternalDataErrorEthereumType(wb, &bchain.Block{
+		BlockHeader: bchain.BlockHeader{
+			Hash:   ie.Hash,
+			Height: ie.Height,
+		},
+	}, ie.ErrorMessage, ie.Retries+1)
+	if err != nil {
+		glog.Errorf("StoreBlockInternalDataErrorEthereumType %d %s, error %v", ie.Height, ie.Hash, err)
+	} else {
+		w.db.WriteBatch(wb)
+	}
+}
+
+func (w *Worker) RefetchInternalDataRoutine() {
+	internalErrors, err := w.db.GetBlockInternalDataErrorsEthereumType()
+	if err == nil {
+		for i := range internalErrors {
+			ie := &internalErrors[i]
+			if ie.Retries >= maxNumberOfRetires {
+				glog.Infof("Refetching internal data for %d %s, retries exceeded", ie.Height, ie.Hash)
+				continue
+			}
+			glog.Infof("Refetching internal data for %d %s, retries %d", ie.Height, ie.Hash, ie.Retries)
+			block, err := w.chain.GetBlock(ie.Hash, ie.Height)
+			var blockSpecificData *bchain.EthereumBlockSpecificData
+			if block != nil {
+				blockSpecificData, _ = block.CoinSpecificData.(*bchain.EthereumBlockSpecificData)
+			}
+			if err != nil || block == nil || (blockSpecificData != nil && blockSpecificData.InternalDataError != "") {
+				glog.Errorf("Refetching internal data for %d %s, error %v, retrying", ie.Height, ie.Hash, err)
+				// try for second time to fetch the data - the 2nd attempt after the first unsuccessful has many times higher probability of success
+				// probably something to do with data preloaded to cache on the backend
+				block, err = w.chain.GetBlock(ie.Hash, ie.Height)
+				if err != nil || block == nil {
+					glog.Errorf("Refetching internal data for %d %s, error %v", ie.Height, ie.Hash, err)
+					continue
+				}
+			}
+			blockSpecificData, _ = block.CoinSpecificData.(*bchain.EthereumBlockSpecificData)
+			if blockSpecificData != nil && blockSpecificData.InternalDataError != "" {
+				glog.Errorf("Refetching internal data for %d %s, internal data error %v", ie.Height, ie.Hash, blockSpecificData.InternalDataError)
+				w.incrementRefetchInternalDataRetryCount(ie)
+			} else {
+				err = w.db.ReconnectInternalDataToBlockEthereumType(block)
+				if err != nil {
+					glog.Errorf("ReconnectInternalDataToBlockEthereumType %d %s, error %v", ie.Height, ie.Hash, err)
+				} else {
+					glog.Infof("Refetching internal data for %d %s, success", ie.Height, ie.Hash)
+				}
+			}
+		}
+	}
+	refetchInternalDataMux.Lock()
+	refetchingInternalData = false
+	refetchInternalDataMux.Unlock()
+}