From 9cbb29f4b77d771ccd445ac16c94f1fe639f7dc1 Mon Sep 17 00:00:00 2001 From: JakubJakubJakub <87699207+Jakub123Jakub123@users.noreply.github.com> Date: Tue, 8 Mar 2022 12:13:37 +0100 Subject: [PATCH 01/19] fix: wallet verification (#168) * fix wallet verification * Remove merkle root references --- electrum/interface.py | 3 ++ electrum/lnverifier.py | 9 ++--- electrum/network.py | 12 +++++- electrum/verifier.py | 84 ++++++++++++------------------------------ 4 files changed, 40 insertions(+), 68 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 1ff366be57eb..199fb2531a94 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -118,6 +118,9 @@ async def send_request(self, *args, timeout=None, **kwargs): except CodeMessageError as e: self.maybe_log(f"--> {repr(e)} (id: {msg_id})") raise + except Exception as e: + self.maybe_log(f"--> {repr(e)} (id: {msg_id})") + raise else: self.maybe_log(f"--> {response} (id: {msg_id})") return response diff --git a/electrum/lnverifier.py b/electrum/lnverifier.py index 3d028363e91e..1335b12195a2 100644 --- a/electrum/lnverifier.py +++ b/electrum/lnverifier.py @@ -34,7 +34,7 @@ from . import constants from .util import bh2u, bfh, NetworkJobOnDefaultServer from .lnutil import funding_output_script_from_keys, ShortChannelID -from .verifier import verify_tx_is_in_block, MerkleVerificationFailure +from .verifier import verify_tx_is_in_block, TransactionVerificationFailure from .transaction import Transaction from .interface import GracefulDisconnect from .crypto import sha256d @@ -110,20 +110,19 @@ async def verify_channel(self, block_height: int, short_channel_id: ShortChannel # we use electrum servers to do this. however we don't trust electrum servers either... try: result = await self.network.get_txid_from_txpos( - block_height, short_channel_id.txpos, True) + block_height, short_channel_id.txpos, False) except aiorpcx.jsonrpc.RPCError: # the electrum server is complaining about the txpos for given block. # it is not clear what to do now, but let's believe the server. self._blacklist_short_channel_id(short_channel_id) return tx_hash = result['tx_hash'] - merkle_branch = result['merkle'] # we need to wait if header sync/reorg is still ongoing, hence lock: async with self.network.bhi_lock: header = self.network.blockchain().read_header(block_height) try: - verify_tx_is_in_block(tx_hash, merkle_branch, short_channel_id.txpos, header, block_height) - except MerkleVerificationFailure as e: + await verify_tx_is_in_block(self.network, tx_hash, short_channel_id.txpos, header, block_height) + except Exception as e: # the electrum server sent an incorrect proof. blame is on server, not the ln peer raise GracefulDisconnect(e) from e try: diff --git a/electrum/network.py b/electrum/network.py index f75190f721a7..384a9fa7b32c 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -1054,10 +1054,18 @@ async def request_chunk(self, height: int, tip=None, *, can_return_early=False): @best_effort_reliable @catch_server_exceptions - async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: + async def get_transaction(self, tx_hash: str, *, timeout=None, verbose=False): if not is_hash256_str(tx_hash): raise Exception(f"{repr(tx_hash)} is not a txid") - return await self.interface.session.send_request('blockchain.transaction.get', [tx_hash], + return await self.interface.session.send_request('blockchain.transaction.get', [tx_hash, verbose], + timeout=timeout) + + @best_effort_reliable + @catch_server_exceptions + async def get_block(self, block_hash: str, *, timeout=None, verbose=False): + if not is_hash256_str(block_hash): + raise Exception(f"{repr(block_hash)} is not a block hash") + return await self.interface.session.send_request('blockchain.block.get_block', [block_hash, verbose], timeout=timeout) @best_effort_reliable diff --git a/electrum/verifier.py b/electrum/verifier.py index d1cdd69c4d2d..b3806f06fe02 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -41,10 +41,10 @@ from .address_synchronizer import AddressSynchronizer -class MerkleVerificationFailure(Exception): pass -class MissingBlockHeader(MerkleVerificationFailure): pass -class MerkleRootMismatch(MerkleVerificationFailure): pass -class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass +class TransactionVerificationFailure(Exception): pass +class MissingBlockHeader(TransactionVerificationFailure): pass +class MerkleRootMismatch(TransactionVerificationFailure): pass +class InnerNodeOfSpvProofIsValidTx(TransactionVerificationFailure): pass class SPV(NetworkJobOnDefaultServer): @@ -118,29 +118,13 @@ async def _mark_as_verified(self, tx_hash, tx_height, tx_type): self.wallet.add_verified_tx(tx_hash, tx_info) async def _request_and_verify_single_proof(self, tx_hash, tx_height, tx_type): - try: - merkle = await self.network.get_merkle_for_transaction(tx_hash, tx_height) - except UntrustedServerReturnedError as e: - if not isinstance(e.original_exception, aiorpcx.jsonrpc.RPCError): - raise - self.logger.info(f'tx {tx_hash} not at height {tx_height}') - self.wallet.remove_unverified_tx(tx_hash, tx_height) - self.requested_merkle.discard(tx_hash) - return - # Verify the hash of the server-provided merkle branch to a - # transaction matches the merkle root of its block - if tx_height != merkle.get('block_height'): - self.logger.info('requested tx_height {} differs from received tx_height {} for txid {}' - .format(tx_height, merkle.get('block_height'), tx_hash)) - tx_height = merkle.get('block_height') - pos = merkle.get('pos') - merkle_branch = merkle.get('merkle') # we need to wait if header sync/reorg is still ongoing, hence lock: + tx_pos = -1 async with self.network.bhi_lock: header = self.network.blockchain().read_header(tx_height) try: - verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height) - except MerkleVerificationFailure as e: + tx_pos = await verify_tx_is_in_block(self.network, tx_hash, header, tx_height) + except TransactionVerificationFailure as e: if self.network.config.get("skipmerklecheck"): self.logger.info(f"skipping merkle proof check {tx_hash}") else: @@ -153,33 +137,11 @@ async def _request_and_verify_single_proof(self, tx_hash, tx_height, tx_type): header_hash = hash_header(header) tx_info = TxMinedInfo(height=tx_height, timestamp=header.get('timestamp'), - txpos=pos, + txpos=tx_pos, header_hash=header_hash, txtype=tx_type) self.wallet.add_verified_tx(tx_hash, tx_info) - @classmethod - def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int): - """Return calculated merkle root.""" - try: - h = hash_decode(tx_hash) - merkle_branch_bytes = [hash_decode(item) for item in merkle_branch] - leaf_pos_in_tree = int(leaf_pos_in_tree) # raise if invalid - except Exception as e: - raise MerkleVerificationFailure(e) - if leaf_pos_in_tree < 0: - raise MerkleVerificationFailure('leaf_pos_in_tree must be non-negative') - index = leaf_pos_in_tree - for item in merkle_branch_bytes: - if len(item) != 32: - raise MerkleVerificationFailure('all merkle branch items have to 32 bytes long') - h = sha256d(item + h) if (index & 1) else sha256d(h + item) - index >>= 1 - cls._raise_if_valid_tx(bh2u(h)) - if index != 0: - raise MerkleVerificationFailure(f'leaf_pos_in_tree too large for branch') - return hash_encode(h) - @classmethod def _raise_if_valid_tx(cls, raw_tx: str): # If an inner node of the merkle proof is also a valid tx, chances are, this is an attack. @@ -214,19 +176,19 @@ def is_up_to_date(self): return not self.requested_merkle -def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str], - leaf_pos_in_tree: int, block_header: Optional[dict], - block_height: int) -> None: - """Raise MerkleVerificationFailure if verification fails.""" +async def verify_tx_is_in_block(network, tx_hash: str, block_header: Optional[dict], + block_height: int): + """Raise TransactionVerificationFailure if verification fails.""" if not block_header: - raise MissingBlockHeader("merkle verification failed for {} (missing header {})" - .format(tx_hash, block_height)) - if len(merkle_branch) > 30: - raise MerkleVerificationFailure(f"merkle branch too long: {len(merkle_branch)}") - calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree) - - block_merkle_root = block_header.get('merkle_root') - # TODO Add method of verifing different than merkle root, because merkle root does not work on current electrumX - # if block_merkle_root != calc_merkle_root: - # raise MerkleRootMismatch("merkle verification failed for {} ({} != {}) height {}".format( - # tx_hash, block_merkle_root, calc_merkle_root, block_height)) + raise MissingBlockHeader("tx verification failed for {} (missing header {})" + .format(tx_hash, block_height)) + try: + res = await network.get_transaction(tx_hash, verbose=True) + + block = await network.get_block(res["blockhash"], verbose=True) + except Exception as e: + print(e) + raise + if block["height"] != block_height or tx_hash not in block["tx"]: + raise TransactionVerificationFailure("Tx is not in block verify_tx_is_in_block()") + return block["tx"].index(tx_hash) From 3da73fc7f17a20a015fc24bf6c29cb1c48e791aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 10:28:45 +0100 Subject: [PATCH 02/19] ci(build): add workflow to prepare artifacts for linux/windows --- .github/workflows/build.yml | 74 +++++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 29 +++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..8701125ef925 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,74 @@ +name: "Build" + +on: + pull_request: + types: + - opened + - synchronize + +jobs: + windows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Builder image + env: + TARGET_OS: Windows + run: docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/ + - name: Compile Windows binary + env: + TARGET_OS: Windows + run: docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: artifact-windows + path: contrib/build-wine/dist/*.exe + if-no-files-found: error + retention-days: 7 + + appimage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Builder image + run: docker build --no-cache -t electrum-appimage-builder-img ./contrib/build-linux/appimage/ + - name: Compile Linux AppImage + run: docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: artifact-appimage + path: dist/*.AppImage + if-no-files-found: error + retention-days: 7 + + tarball: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Builder image + run: docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/ + - name: Compile Linux tarball + run: | + echo "Building sdist at $PWD" + docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: artifact-tarball + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000000..8f228825b587 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: "Test" + +on: + pull_request: + types: + - opened + - synchronize + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install system dependencies + run: | + sudo apt-get -qq update + sudo apt-get install -yq libsecp256k1-0 + - name: Install dependencies + run: | + pip install -r contrib/requirements/requirements.txt + pip install electrumx flake8 + - name: Flake8 linter tests + run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Unittests + run: python -m unittest -v electrum/tests/test_*.py \ No newline at end of file From 1b9d0ee39a4d645df8f75bc67af1501d4697bdd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 11:40:08 +0200 Subject: [PATCH 03/19] build(docker): merged changes from remote --- .github/workflows/test.yml | 29 ------------------------- .gitignore | 14 ++++++++++++ contrib/build-linux/appimage/Dockerfile | 5 ++++- contrib/build-wine/Dockerfile | 8 +++++-- 4 files changed, 24 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 8f228825b587..000000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: "Test" - -on: - pull_request: - types: - - opened - - synchronize - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install system dependencies - run: | - sudo apt-get -qq update - sudo apt-get install -yq libsecp256k1-0 - - name: Install dependencies - run: | - pip install -r contrib/requirements/requirements.txt - pip install electrumx flake8 - - name: Flake8 linter tests - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - - name: Unittests - run: python -m unittest -v electrum/tests/test_*.py \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f0c77ef1dca..a9d62b38f7ad 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,20 @@ venv .DS_Store # Temporary bulds files +contrib/secp256k1 contrib/osx/libsecp256k1.0.dylib contrib/osx/libusb-1.0.dylib +electrum/libsecp256k1-0.dll +electrum/libsecp256k1.so.0 prebuilt_qr/ + +# generated locale's +contrib/deterministic-build/electrum-locale +electrum/es_ES +electrum/id_ID +electrum/ja_JP +electrum/ko_KR +electrum/pt_PT +electrum/tr_TR +electrum/vi_VN +electrum/zh_CN diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index 7003235a2694..29095b630e48 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -21,11 +21,14 @@ RUN apt-get update -q && \ libusb-1.0-0-dev=2:1.0.20-1 \ libudev-dev=229-4ubuntu21.31 \ gettext=0.19.7-2ubuntu3.1 \ - libzbar0=0.10+doc-10ubuntu1 \ + libzbar0=0.10+doc-10ubuntu1 \ libdbus-1-3=1.10.6-1ubuntu3.6 \ libxkbcommon-x11-0=0.5.0-1ubuntu2.1 \ +<<<<<<< HEAD libxcb-util1=0.4.0-0ubuntu3 \ libxcb-xinerama0=1.11.1-1ubuntu1 \ +======= +>>>>>>> 674b801c (ci(build): update lib dependences to correct versions) libc6-dev=2.23-0ubuntu11.3 \ && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 60464e2cde4e..54ccec9103d5 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -1,4 +1,3 @@ - FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 @@ -9,8 +8,13 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.2 \ gnupg2=2.2.4-1ubuntu1.4 \ dirmngr=2.2.4-1ubuntu1.4 \ +<<<<<<< HEAD python3-software-properties=0.96.24.32.1 \ software-properties-common=0.96.24.32.1 +======= + python3-software-properties=0.96.24.32.14 \ + software-properties-common=0.96.24.32.14 +>>>>>>> 674b801c (ci(build): update lib dependences to correct versions) RUN apt-get update -q && \ apt-get install -qy \ @@ -24,7 +28,7 @@ RUN apt-get update -q && \ autopoint=0.19.8.1-6ubuntu0.3 \ autoconf=2.69-11 \ libtool=2.4.6-2 \ - gettext=0.19.8.1-6 + gettext=0.19.8.1-6ubuntu0.3 RUN wget -nc https://dl.winehq.org/wine-builds/Release.key && \ echo "c51bcb8cc4a12abfbd7c7660eaf90f49674d15e222c262f27e6c96429111b822 Release.key" | sha256sum -c - && \ From 08e01aef9f8441d7823709ea7460dc20543f34bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 11:04:55 +0100 Subject: [PATCH 04/19] ci(build): removed unused tarball build --- .github/workflows/build.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8701125ef925..2a0c86764e50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,25 +50,3 @@ jobs: path: dist/*.AppImage if-no-files-found: error retention-days: 7 - - tarball: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Builder image - run: docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/ - - name: Compile Linux tarball - run: | - echo "Building sdist at $PWD" - docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh - - name: 'Upload Artifact' - uses: actions/upload-artifact@v2 - with: - name: artifact-tarball - path: dist/*.tar.gz - if-no-files-found: error - retention-days: 7 From 058b7b29eb4d5945ee7d312e49c24d59e34ce8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 14:03:22 +0100 Subject: [PATCH 05/19] build(osx): add poc --- .github/workflows/build.yml | 83 +++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a0c86764e50..32b2fcc55122 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,47 +6,58 @@ on: - opened - synchronize + +env: + BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_BUILD_CERTIFICATE_BASE64 }} + P12_PASSWORD: ${{ secrets.OSX_P12_PASSWORD }} + #BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }} + KEYCHAIN_PASSWORD: ${{ secrets.OSX_KEYCHAIN_PASSWORD }} + CODESIGN_CERT: ${{ secrets.OSX_CODESIGN_CERT }} + APPLE_ID_USER: ${{ secrets.OSX_APPLE_ID_USER }} + APPLE_ID_PASSWORD: ${{ secrets.OSX_APPLE_ID_PASSWORD }} + APPLE_ID_PROVIDER_SHORT_NAME: ${{ secrets.OSX_APPLE_ID_PROVIDER_SHORT_NAME }} + jobs: - windows: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Builder image - env: - TARGET_OS: Windows - run: docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/ - - name: Compile Windows binary - env: - TARGET_OS: Windows - run: docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh - - name: 'Upload Artifact' - uses: actions/upload-artifact@v2 - with: - name: artifact-windows - path: contrib/build-wine/dist/*.exe - if-no-files-found: error - retention-days: 7 + macos: + runs-on: macos-latest - appimage: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Builder image - run: docker build --no-cache -t electrum-appimage-builder-img ./contrib/build-linux/appimage/ - - name: Compile Linux AppImage - run: docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install the Apple certificate + run: | + # create variables + CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12 + #PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision + KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db + + # import certificate and provisioning profile from secrets + echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode --output $CERTIFICATE_PATH + #echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode --output $PP_PATH + + # create temporary keychain + security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH + security set-keychain-settings -lut 21600 $KEYCHAIN_PATH + security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH + + # import certificate to keychain + security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH + security list-keychain -d user -s $KEYCHAIN_PATH + + # apply provisioning profile + #mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles + #cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles + - name: Build app + run: ./contrib/osx/make_osx $CODESIGN_CERT + - name: Debug + run: | + pwd + ls -la + ls -la dist - name: 'Upload Artifact' uses: actions/upload-artifact@v2 with: - name: artifact-appimage - path: dist/*.AppImage + name: artifact-macos + path: dist/*.dmg if-no-files-found: error retention-days: 7 From 04fdae4a0823dcc3de6851312060a90374dabf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 14:05:29 +0100 Subject: [PATCH 06/19] build(osx): run workflow on code push --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32b2fcc55122..49473eef05a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: "Build" on: + push: pull_request: types: - opened From 5d75771fca92601864a912b6ab686fa03c3fb9a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 14:35:25 +0100 Subject: [PATCH 07/19] build(osx): add scripts to build missing libs --- contrib/make_libusb.sh | 63 ++++++++++++++++++++++++++++ contrib/make_zbar.sh | 93 ++++++++++++++++++++++++++++++++++++++++++ contrib/osx/make_osx | 36 +++++++++++----- 3 files changed, 181 insertions(+), 11 deletions(-) create mode 100755 contrib/make_libusb.sh create mode 100755 contrib/make_zbar.sh diff --git a/contrib/make_libusb.sh b/contrib/make_libusb.sh new file mode 100755 index 000000000000..20e3f27c375e --- /dev/null +++ b/contrib/make_libusb.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +LIBUSB_VERSION="c6a35c56016ea2ab2f19115d2ea1e85e0edae155" +# ^ tag v1.0.24 + +set -e + +. $(dirname "$0")/build_tools_util.sh || (echo "Could not source build_tools_util.sh" && exit 1) + +here=$(dirname $(realpath "$0" 2> /dev/null || grealpath "$0")) +CONTRIB="$here" +PROJECT_ROOT="$CONTRIB/.." + +pkgname="libusb" +info "Building $pkgname..." + +( + cd $CONTRIB + if [ ! -d libusb ]; then + git clone https://github.com/libusb/libusb.git + fi + cd libusb + if ! $(git cat-file -e ${LIBUSB_VERSION}) ; then + info "Could not find requested version $LIBUSB_VERSION in local clone; fetching..." + git fetch --all + fi + git reset --hard + git clean -dfxq + git checkout "${LIBUSB_VERSION}^{commit}" + + if [ "$BUILD_TYPE" = "wine" ] ; then + echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am + fi + ./bootstrap.sh || fail "Could not bootstrap libusb" + if ! [ -r config.status ] ; then + if [ "$BUILD_TYPE" = "wine" ] ; then + # windows target + LDFLAGS="-Wl,--no-insert-timestamp" + elif [ $(uname) == "Darwin" ]; then + # macos target + LDFLAGS="-Wl -lm" + else + # linux target + LDFLAGS="" + fi + LDFLAGS="$LDFLAGS" ./configure \ + $AUTOCONF_FLAGS \ + || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again." + fi + make -j4 || fail "Could not build $pkgname" + make install || fail "Could not install $pkgname" + . "$here/$pkgname/libusb/.libs/libusb-1.0.la" + host_strip "$here/$pkgname/libusb/.libs/$dlname" + TARGET_NAME="$dlname" + if [ $(uname) == "Darwin" ]; then # on mac, dlname is "libusb-1.0.0.dylib" + TARGET_NAME="libusb-1.0.dylib" + fi + cp -fpv "$here/$pkgname/libusb/.libs/$dlname" "$PROJECT_ROOT/electrum/$TARGET_NAME" || fail "Could not copy the $pkgname binary to its destination" + info "$TARGET_NAME has been placed in the inner 'electrum' folder." + if [ -n "$DLL_TARGET_DIR" ] ; then + cp -fpv "$here/$pkgname/libusb/.libs/$dlname" "$DLL_TARGET_DIR/$TARGET_NAME" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" + fi +) diff --git a/contrib/make_zbar.sh b/contrib/make_zbar.sh new file mode 100755 index 000000000000..de60c68abd6e --- /dev/null +++ b/contrib/make_zbar.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script can be used on Linux hosts to build native libzbar binaries. +# sudo apt-get install pkg-config libx11-dev libx11-6 libv4l-dev libxv-dev libxext-dev libjpeg-dev +# +# It can also be used to cross-compile to Windows: +# $ sudo apt-get install mingw-w64 mingw-w64-tools win-iconv-mingw-w64-dev +# For a Windows x86 (32-bit) target, run: +# $ GCC_TRIPLET_HOST="i686-w64-mingw32" BUILD_TYPE="wine" ./contrib/make_zbar.sh +# Or for a Windows x86_64 (64-bit) target, run: +# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" BUILD_TYPE="wine" ./contrib/make_zbar.sh + +ZBAR_VERSION="aac86d5f08d64ab4c3da78188eb622fa3cb07182" + +set -e + +. $(dirname "$0")/build_tools_util.sh || (echo "Could not source build_tools_util.sh" && exit 1) + +here=$(dirname $(realpath "$0" 2> /dev/null || grealpath "$0")) +CONTRIB="$here" +PROJECT_ROOT="$CONTRIB/.." + +pkgname="zbar" +info "Building $pkgname..." + +( + cd $CONTRIB + if [ ! -d zbar ]; then + git clone https://github.com/mchehab/zbar.git + fi + cd zbar + if ! $(git cat-file -e ${ZBAR_VERSION}) ; then + info "Could not find requested version $ZBAR_VERSION in local clone; fetching..." + git fetch --all + fi + git reset --hard + git clean -dfxq + git checkout "${ZBAR_VERSION}^{commit}" + + if [ "$BUILD_TYPE" = "wine" ] ; then + echo "libzbar_la_LDFLAGS += -Wc,-static" >> zbar/Makefile.am + echo "LDFLAGS += -Wc,-static" >> Makefile.am + fi + if ! [ -x configure ] ; then + autoreconf -vfi || fail "Could not run autoreconf for $pkgname. Please make sure you have automake and libtool installed, and try again." + fi + if ! [ -r config.status ] ; then + if [ "$BUILD_TYPE" = "wine" ] ; then + # windows target + AUTOCONF_FLAGS="$AUTOCONF_FLAGS \ + --with-x=no \ + --enable-video=yes \ + --with-jpeg=no \ + --with-directshow=yes \ + --disable-dependency-tracking" + elif [ $(uname) == "Darwin" ]; then + # macos target + AUTOCONF_FLAGS="$AUTOCONF_FLAGS \ + --with-x=no \ + --enable-video=no \ + --with-jpeg=no" + else + # linux target + AUTOCONF_FLAGS="$AUTOCONF_FLAGS \ + --with-x=yes \ + --enable-video=yes \ + --with-jpeg=yes" + fi + ./configure \ + $AUTOCONF_FLAGS \ + --prefix="$here/$pkgname/dist" \ + --enable-pthread=no \ + --enable-doc=no \ + --with-python=no \ + --with-gtk=no \ + --with-qt=no \ + --with-java=no \ + --with-imagemagick=no \ + --with-dbus=no \ + --enable-codes=qrcode \ + --disable-static \ + --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again." + fi + make -j4 || fail "Could not build $pkgname" + make install || fail "Could not install $pkgname" + . "$here/$pkgname/dist/lib/libzbar.la" + host_strip "$here/$pkgname/dist/lib/$dlname" + cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination" + info "$dlname has been placed in the inner 'electrum' folder." + if [ -n "$DLL_TARGET_DIR" ] ; then + cp -fpv "$here/$pkgname/dist/lib/$dlname" "$DLL_TARGET_DIR" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" + fi +) diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx index c193be963bb5..62c64b5c3948 100755 --- a/contrib/osx/make_osx +++ b/contrib/osx/make_osx @@ -114,18 +114,32 @@ info "generating locale" done ) || fail "failed generating locale" +info "Installing some build-time deps for compilation..." +brew install autoconf automake libtool gettext coreutils pkgconfig -info "Downloading libusb..." -curl https://homebrew.bintray.com/bottles/libusb-1.0.23.high_sierra.bottle.tar.gz | \ -tar xz --directory "$BUILDDIR" -cp "$BUILDDIR/libusb/1.0.23/lib/libusb-1.0.dylib" contrib/osx -echo "caea266f3fc3982adc55d6cb8d9bad10f6e61f0c24ce5901aa1804618e08e14d contrib/osx/libusb-1.0.dylib" | \ - shasum -a 256 -c || fail "libusb checksum mismatched" - -info "Preparing for building libsecp256k1" -brew install autoconf automake libtool -"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" -cp "$ROOT_FOLDER"/electrum/libsecp256k1.0.dylib contrib/osx +if [ ! -f "$PROJECT_ROOT"/electrum/libsecp256k1.0.dylib ]; then + info "Building libsecp256k1 dylib..." + "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" +else + info "Skipping libsecp256k1 build: reusing already built dylib." +fi +cp "$PROJECT_ROOT"/electrum/libsecp256k1.0.dylib "$CONTRIB"/osx + +if [ ! -f "$PROJECT_ROOT"/electrum/libzbar.0.dylib ]; then + info "Building ZBar dylib..." + "$CONTRIB"/make_zbar.sh || fail "Could not build ZBar dylib" +else + info "Skipping ZBar build: reusing already built dylib." +fi +cp "$PROJECT_ROOT"/electrum/libzbar.0.dylib "$CONTRIB"/osx + +if [ ! -f "$PROJECT_ROOT"/electrum/libusb-1.0.dylib ]; then + info "Building libusb dylib..." + "$CONTRIB"/make_libusb.sh || fail "Could not build libusb dylib" +else + info "Skipping libusb build: reusing already built dylib." +fi +cp "$PROJECT_ROOT"/electrum/libusb-1.0.dylib "$CONTRIB"/osx info "Building CalinsQRReader..." d=contrib/osx/CalinsQRReader From 30b5b8416fa5c86730fe3f1797d18682f44014d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 15:03:02 +0100 Subject: [PATCH 08/19] build(osx): add openssl lib --- contrib/osx/make_osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx index 62c64b5c3948..f9de158945c2 100755 --- a/contrib/osx/make_osx +++ b/contrib/osx/make_osx @@ -115,7 +115,7 @@ info "generating locale" ) || fail "failed generating locale" info "Installing some build-time deps for compilation..." -brew install autoconf automake libtool gettext coreutils pkgconfig +brew install autoconf automake libtool gettext coreutils pkgconfig openssl if [ ! -f "$PROJECT_ROOT"/electrum/libsecp256k1.0.dylib ]; then info "Building libsecp256k1 dylib..." From 85df645e380ead39871b3ad90f43928fc49b7dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 15:29:11 +0100 Subject: [PATCH 09/19] build(osx): use Catalina for builds --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49473eef05a9..6fefcba7fb76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ env: jobs: macos: - runs-on: macos-latest + runs-on: macos-10.15 steps: - name: Checkout repository From 981d7a3c230e467c8f5b1e82185e8280f868e31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 16:13:45 +0100 Subject: [PATCH 10/19] build(osx): fix project root path --- contrib/osx/make_osx | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx index f9de158945c2..43f6e00d812a 100755 --- a/contrib/osx/make_osx +++ b/contrib/osx/make_osx @@ -14,6 +14,7 @@ export GCC_STRIP_BINARIES="1" CONTRIB_OSX="$(dirname "$(realpath "$0")")" CONTRIB="$CONTRIB_OSX/.." +PROJECT_ROOT="$CONTRIB/.." ROOT_FOLDER="$CONTRIB/.." src_dir=$(dirname "$0") From 21433a8d52461d0e37672e8cb53fd7c22a79262c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Fri, 18 Mar 2022 16:37:24 +0100 Subject: [PATCH 11/19] build(osx): remove debug lines --- .github/workflows/build.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6fefcba7fb76..58e38e125f42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,17 +1,14 @@ name: "Build" on: - push: pull_request: types: - opened - synchronize - env: BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_BUILD_CERTIFICATE_BASE64 }} P12_PASSWORD: ${{ secrets.OSX_P12_PASSWORD }} - #BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }} KEYCHAIN_PASSWORD: ${{ secrets.OSX_KEYCHAIN_PASSWORD }} CODESIGN_CERT: ${{ secrets.OSX_CODESIGN_CERT }} APPLE_ID_USER: ${{ secrets.OSX_APPLE_ID_USER }} @@ -29,12 +26,10 @@ jobs: run: | # create variables CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12 - #PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db # import certificate and provisioning profile from secrets echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode --output $CERTIFICATE_PATH - #echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode --output $PP_PATH # create temporary keychain security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH @@ -44,17 +39,8 @@ jobs: # import certificate to keychain security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH security list-keychain -d user -s $KEYCHAIN_PATH - - # apply provisioning profile - #mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles - #cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles - name: Build app run: ./contrib/osx/make_osx $CODESIGN_CERT - - name: Debug - run: | - pwd - ls -la - ls -la dist - name: 'Upload Artifact' uses: actions/upload-artifact@v2 with: From 76f2184c597d68c958c1cf14b9da2778d469b6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 11:40:54 +0200 Subject: [PATCH 12/19] build(docker): merged changes from remote --- .github/workflows/build.yml | 45 +++++++++++++++++++++++++ contrib/build-linux/appimage/Dockerfile | 3 ++ contrib/build-wine/Dockerfile | 5 +++ 3 files changed, 53 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58e38e125f42..8cc0892f8041 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,7 @@ on: - opened - synchronize +<<<<<<< HEAD env: BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_BUILD_CERTIFICATE_BASE64 }} P12_PASSWORD: ${{ secrets.OSX_P12_PASSWORD }} @@ -46,5 +47,49 @@ jobs: with: name: artifact-macos path: dist/*.dmg +======= +jobs: + windows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Builder image + env: + TARGET_OS: Windows + run: docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/ + - name: Compile Windows binary + env: + TARGET_OS: Windows + run: docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: artifact-windows + path: contrib/build-wine/dist/*.exe + if-no-files-found: error + retention-days: 7 + + appimage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Builder image + run: docker build --no-cache -t electrum-appimage-builder-img ./contrib/build-linux/appimage/ + - name: Compile Linux AppImage + run: docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: artifact-appimage + path: dist/*.AppImage +>>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) if-no-files-found: error retention-days: 7 diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index 29095b630e48..d357b33078ec 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -24,11 +24,14 @@ RUN apt-get update -q && \ libzbar0=0.10+doc-10ubuntu1 \ libdbus-1-3=1.10.6-1ubuntu3.6 \ libxkbcommon-x11-0=0.5.0-1ubuntu2.1 \ +<<<<<<< HEAD <<<<<<< HEAD libxcb-util1=0.4.0-0ubuntu3 \ libxcb-xinerama0=1.11.1-1ubuntu1 \ ======= >>>>>>> 674b801c (ci(build): update lib dependences to correct versions) +======= +>>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) libc6-dev=2.23-0ubuntu11.3 \ && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 54ccec9103d5..e2f998d912ca 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -8,6 +8,7 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.2 \ gnupg2=2.2.4-1ubuntu1.4 \ dirmngr=2.2.4-1ubuntu1.4 \ +<<<<<<< HEAD <<<<<<< HEAD python3-software-properties=0.96.24.32.1 \ software-properties-common=0.96.24.32.1 @@ -15,6 +16,10 @@ RUN dpkg --add-architecture i386 && \ python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 >>>>>>> 674b801c (ci(build): update lib dependences to correct versions) +======= + python3-software-properties=0.96.24.32.14 \ + software-properties-common=0.96.24.32.14 +>>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) RUN apt-get update -q && \ apt-get install -qy \ From 5b434b81da518b71e89ca770690f4345120ee83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 11:45:14 +0200 Subject: [PATCH 13/19] ci: fix rebase --- .github/workflows/build.yml | 5 +---- contrib/build-linux/appimage/Dockerfile | 6 ------ contrib/build-wine/Dockerfile | 6 ------ 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8cc0892f8041..60ff6fa11c7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,6 @@ on: - opened - synchronize -<<<<<<< HEAD env: BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_BUILD_CERTIFICATE_BASE64 }} P12_PASSWORD: ${{ secrets.OSX_P12_PASSWORD }} @@ -47,8 +46,7 @@ jobs: with: name: artifact-macos path: dist/*.dmg -======= -jobs: + windows: runs-on: ubuntu-latest steps: @@ -90,6 +88,5 @@ jobs: with: name: artifact-appimage path: dist/*.AppImage ->>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) if-no-files-found: error retention-days: 7 diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index d357b33078ec..f4afb1e9fd97 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -24,14 +24,8 @@ RUN apt-get update -q && \ libzbar0=0.10+doc-10ubuntu1 \ libdbus-1-3=1.10.6-1ubuntu3.6 \ libxkbcommon-x11-0=0.5.0-1ubuntu2.1 \ -<<<<<<< HEAD -<<<<<<< HEAD libxcb-util1=0.4.0-0ubuntu3 \ libxcb-xinerama0=1.11.1-1ubuntu1 \ -======= ->>>>>>> 674b801c (ci(build): update lib dependences to correct versions) -======= ->>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) libc6-dev=2.23-0ubuntu11.3 \ && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index e2f998d912ca..3a3b72d20daa 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -8,18 +8,12 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.2 \ gnupg2=2.2.4-1ubuntu1.4 \ dirmngr=2.2.4-1ubuntu1.4 \ -<<<<<<< HEAD -<<<<<<< HEAD python3-software-properties=0.96.24.32.1 \ software-properties-common=0.96.24.32.1 -======= python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 ->>>>>>> 674b801c (ci(build): update lib dependences to correct versions) -======= python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 ->>>>>>> e0d5305f (ci(build): add workflow to prepare artifacts for linux/windows (#169)) RUN apt-get update -q && \ apt-get install -qy \ From 476d6815043b4ba4a1c340579b101b41cce360a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 11:48:09 +0200 Subject: [PATCH 14/19] build: fix dockerfile --- contrib/build-wine/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 3a3b72d20daa..3e6b0046a500 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -8,10 +8,6 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.2 \ gnupg2=2.2.4-1ubuntu1.4 \ dirmngr=2.2.4-1ubuntu1.4 \ - python3-software-properties=0.96.24.32.1 \ - software-properties-common=0.96.24.32.1 - python3-software-properties=0.96.24.32.14 \ - software-properties-common=0.96.24.32.14 python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 From c39e93b987d46857628b03c1525638efc8bdcba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 11:48:40 +0200 Subject: [PATCH 15/19] Revert "fix: wallet verification (#168)" This reverts commit 9cbb29f4b77d771ccd445ac16c94f1fe639f7dc1. --- electrum/interface.py | 3 -- electrum/lnverifier.py | 9 +++-- electrum/network.py | 12 +----- electrum/verifier.py | 84 ++++++++++++++++++++++++++++++------------ 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 199fb2531a94..1ff366be57eb 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -118,9 +118,6 @@ async def send_request(self, *args, timeout=None, **kwargs): except CodeMessageError as e: self.maybe_log(f"--> {repr(e)} (id: {msg_id})") raise - except Exception as e: - self.maybe_log(f"--> {repr(e)} (id: {msg_id})") - raise else: self.maybe_log(f"--> {response} (id: {msg_id})") return response diff --git a/electrum/lnverifier.py b/electrum/lnverifier.py index 1335b12195a2..3d028363e91e 100644 --- a/electrum/lnverifier.py +++ b/electrum/lnverifier.py @@ -34,7 +34,7 @@ from . import constants from .util import bh2u, bfh, NetworkJobOnDefaultServer from .lnutil import funding_output_script_from_keys, ShortChannelID -from .verifier import verify_tx_is_in_block, TransactionVerificationFailure +from .verifier import verify_tx_is_in_block, MerkleVerificationFailure from .transaction import Transaction from .interface import GracefulDisconnect from .crypto import sha256d @@ -110,19 +110,20 @@ async def verify_channel(self, block_height: int, short_channel_id: ShortChannel # we use electrum servers to do this. however we don't trust electrum servers either... try: result = await self.network.get_txid_from_txpos( - block_height, short_channel_id.txpos, False) + block_height, short_channel_id.txpos, True) except aiorpcx.jsonrpc.RPCError: # the electrum server is complaining about the txpos for given block. # it is not clear what to do now, but let's believe the server. self._blacklist_short_channel_id(short_channel_id) return tx_hash = result['tx_hash'] + merkle_branch = result['merkle'] # we need to wait if header sync/reorg is still ongoing, hence lock: async with self.network.bhi_lock: header = self.network.blockchain().read_header(block_height) try: - await verify_tx_is_in_block(self.network, tx_hash, short_channel_id.txpos, header, block_height) - except Exception as e: + verify_tx_is_in_block(tx_hash, merkle_branch, short_channel_id.txpos, header, block_height) + except MerkleVerificationFailure as e: # the electrum server sent an incorrect proof. blame is on server, not the ln peer raise GracefulDisconnect(e) from e try: diff --git a/electrum/network.py b/electrum/network.py index 384a9fa7b32c..f75190f721a7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -1054,18 +1054,10 @@ async def request_chunk(self, height: int, tip=None, *, can_return_early=False): @best_effort_reliable @catch_server_exceptions - async def get_transaction(self, tx_hash: str, *, timeout=None, verbose=False): + async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: if not is_hash256_str(tx_hash): raise Exception(f"{repr(tx_hash)} is not a txid") - return await self.interface.session.send_request('blockchain.transaction.get', [tx_hash, verbose], - timeout=timeout) - - @best_effort_reliable - @catch_server_exceptions - async def get_block(self, block_hash: str, *, timeout=None, verbose=False): - if not is_hash256_str(block_hash): - raise Exception(f"{repr(block_hash)} is not a block hash") - return await self.interface.session.send_request('blockchain.block.get_block', [block_hash, verbose], + return await self.interface.session.send_request('blockchain.transaction.get', [tx_hash], timeout=timeout) @best_effort_reliable diff --git a/electrum/verifier.py b/electrum/verifier.py index b3806f06fe02..d1cdd69c4d2d 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -41,10 +41,10 @@ from .address_synchronizer import AddressSynchronizer -class TransactionVerificationFailure(Exception): pass -class MissingBlockHeader(TransactionVerificationFailure): pass -class MerkleRootMismatch(TransactionVerificationFailure): pass -class InnerNodeOfSpvProofIsValidTx(TransactionVerificationFailure): pass +class MerkleVerificationFailure(Exception): pass +class MissingBlockHeader(MerkleVerificationFailure): pass +class MerkleRootMismatch(MerkleVerificationFailure): pass +class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass class SPV(NetworkJobOnDefaultServer): @@ -118,13 +118,29 @@ async def _mark_as_verified(self, tx_hash, tx_height, tx_type): self.wallet.add_verified_tx(tx_hash, tx_info) async def _request_and_verify_single_proof(self, tx_hash, tx_height, tx_type): + try: + merkle = await self.network.get_merkle_for_transaction(tx_hash, tx_height) + except UntrustedServerReturnedError as e: + if not isinstance(e.original_exception, aiorpcx.jsonrpc.RPCError): + raise + self.logger.info(f'tx {tx_hash} not at height {tx_height}') + self.wallet.remove_unverified_tx(tx_hash, tx_height) + self.requested_merkle.discard(tx_hash) + return + # Verify the hash of the server-provided merkle branch to a + # transaction matches the merkle root of its block + if tx_height != merkle.get('block_height'): + self.logger.info('requested tx_height {} differs from received tx_height {} for txid {}' + .format(tx_height, merkle.get('block_height'), tx_hash)) + tx_height = merkle.get('block_height') + pos = merkle.get('pos') + merkle_branch = merkle.get('merkle') # we need to wait if header sync/reorg is still ongoing, hence lock: - tx_pos = -1 async with self.network.bhi_lock: header = self.network.blockchain().read_header(tx_height) try: - tx_pos = await verify_tx_is_in_block(self.network, tx_hash, header, tx_height) - except TransactionVerificationFailure as e: + verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height) + except MerkleVerificationFailure as e: if self.network.config.get("skipmerklecheck"): self.logger.info(f"skipping merkle proof check {tx_hash}") else: @@ -137,11 +153,33 @@ async def _request_and_verify_single_proof(self, tx_hash, tx_height, tx_type): header_hash = hash_header(header) tx_info = TxMinedInfo(height=tx_height, timestamp=header.get('timestamp'), - txpos=tx_pos, + txpos=pos, header_hash=header_hash, txtype=tx_type) self.wallet.add_verified_tx(tx_hash, tx_info) + @classmethod + def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int): + """Return calculated merkle root.""" + try: + h = hash_decode(tx_hash) + merkle_branch_bytes = [hash_decode(item) for item in merkle_branch] + leaf_pos_in_tree = int(leaf_pos_in_tree) # raise if invalid + except Exception as e: + raise MerkleVerificationFailure(e) + if leaf_pos_in_tree < 0: + raise MerkleVerificationFailure('leaf_pos_in_tree must be non-negative') + index = leaf_pos_in_tree + for item in merkle_branch_bytes: + if len(item) != 32: + raise MerkleVerificationFailure('all merkle branch items have to 32 bytes long') + h = sha256d(item + h) if (index & 1) else sha256d(h + item) + index >>= 1 + cls._raise_if_valid_tx(bh2u(h)) + if index != 0: + raise MerkleVerificationFailure(f'leaf_pos_in_tree too large for branch') + return hash_encode(h) + @classmethod def _raise_if_valid_tx(cls, raw_tx: str): # If an inner node of the merkle proof is also a valid tx, chances are, this is an attack. @@ -176,19 +214,19 @@ def is_up_to_date(self): return not self.requested_merkle -async def verify_tx_is_in_block(network, tx_hash: str, block_header: Optional[dict], - block_height: int): - """Raise TransactionVerificationFailure if verification fails.""" +def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str], + leaf_pos_in_tree: int, block_header: Optional[dict], + block_height: int) -> None: + """Raise MerkleVerificationFailure if verification fails.""" if not block_header: - raise MissingBlockHeader("tx verification failed for {} (missing header {})" - .format(tx_hash, block_height)) - try: - res = await network.get_transaction(tx_hash, verbose=True) - - block = await network.get_block(res["blockhash"], verbose=True) - except Exception as e: - print(e) - raise - if block["height"] != block_height or tx_hash not in block["tx"]: - raise TransactionVerificationFailure("Tx is not in block verify_tx_is_in_block()") - return block["tx"].index(tx_hash) + raise MissingBlockHeader("merkle verification failed for {} (missing header {})" + .format(tx_hash, block_height)) + if len(merkle_branch) > 30: + raise MerkleVerificationFailure(f"merkle branch too long: {len(merkle_branch)}") + calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree) + + block_merkle_root = block_header.get('merkle_root') + # TODO Add method of verifing different than merkle root, because merkle root does not work on current electrumX + # if block_merkle_root != calc_merkle_root: + # raise MerkleRootMismatch("merkle verification failed for {} ({} != {}) height {}".format( + # tx_hash, block_merkle_root, calc_merkle_root, block_height)) From 30b2a0108c1042fde345bacc424b30aa0be457a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 12:07:10 +0200 Subject: [PATCH 16/19] build(dockerfile): update depts --- contrib/build-wine/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 3e6b0046a500..82cbd3f4bbba 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -6,14 +6,14 @@ RUN dpkg --add-architecture i386 && \ apt-get update -q && \ apt-get install -qy \ wget=1.19.4-1ubuntu2.2 \ - gnupg2=2.2.4-1ubuntu1.4 \ - dirmngr=2.2.4-1ubuntu1.4 \ + gnupg2 \ + dirmngr \ python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 RUN apt-get update -q && \ apt-get install -qy \ - git=1:2.17.1-1ubuntu0.9 \ + git \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ From 300081e7c15a66b8c934b7fb5621998cd2a5f395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 12:11:29 +0200 Subject: [PATCH 17/19] build(dockerfile): update depts --- contrib/build-wine/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 82cbd3f4bbba..27e742c32da7 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -6,14 +6,14 @@ RUN dpkg --add-architecture i386 && \ apt-get update -q && \ apt-get install -qy \ wget=1.19.4-1ubuntu2.2 \ - gnupg2 \ - dirmngr \ + gnupg2=2.2.4-1ubuntu1.5 \ + dirmngr=2.2.4-1ubuntu1.5 \ python3-software-properties=0.96.24.32.14 \ software-properties-common=0.96.24.32.14 RUN apt-get update -q && \ apt-get install -qy \ - git \ + git=1:2.17.1-1ubuntu0.11 \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ From 4f368b8300a62fdb8f6b694346d3a393dac20f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 12:35:14 +0200 Subject: [PATCH 18/19] build(dockerfile): update depts --- contrib/build-wine/build-electrum-git.sh | 3 ++- contrib/build-wine/deterministic.spec | 9 +++------ contrib/build-wine/prepare-wine.sh | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh index e7ec10155752..fff7f9677149 100755 --- a/contrib/build-wine/build-electrum-git.sh +++ b/contrib/build-wine/build-electrum-git.sh @@ -11,6 +11,7 @@ export PYTHONHASHSEED=22 PYHOME=c:/python3 PYTHON="wine $PYHOME/python.exe -OO -B" +git config --global --add safe.directory $WINEPREFIX/drive_c/electrum # Let's begin! set -e @@ -60,7 +61,7 @@ rm -rf dist/ # build standalone and portable versions info "Running pyinstaller..." -wine "$PYHOME/scripts/pyinstaller.exe" --noconfirm --ascii --clean --name $NAME_ROOT-$VERSION -w deterministic.spec +wine "$PYHOME/scripts/pyinstaller.exe" --noconfirm --ascii --clean deterministic.spec # set timestamps in dist, in order to make the installer reproducible pushd dist diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index c4db5367fb30..176f772f7560 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -3,12 +3,9 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_dynamic_libs import sys -for i, x in enumerate(sys.argv): - if x == '--name': - cmdline_name = sys.argv[i+1] - break -else: - raise Exception('no name') +for i, x in enumerate(sys.argv):import os + +cmdline_name = str(os.environ['NAME_ROOT']+"-"+os.environ['VERSION']) PYHOME = 'c:/python3' diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh index ef8ea1fab540..1b1ce8e53600 100755 --- a/contrib/build-wine/prepare-wine.sh +++ b/contrib/build-wine/prepare-wine.sh @@ -123,6 +123,6 @@ info "Building PyInstaller." fi ) || fail "PyInstaller build failed" info "Installing PyInstaller." -$PYTHON -m pip install --no-dependencies --no-warn-script-location ./pyinstaller +$PYTHON -m pip install pyinstaller #--no-dependencies --no-warn-script-location ./pyinstaller info "Wine is configured." From c118181f20c18a65611bd95fc26c811619b991f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= Date: Tue, 14 Jun 2022 13:51:25 +0200 Subject: [PATCH 19/19] build(dockerfile): update depts --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60ff6fa11c7a..cda801d9a879 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,7 @@ jobs: uses: actions/checkout@v2 - name: Install the Apple certificate run: | + set -e # create variables CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12 KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db @@ -37,7 +38,7 @@ jobs: security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH # import certificate to keychain - security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH + security import $CERTIFICATE_PATH -P $P12_PASSWORD -A -t cert -f pkcs12 -k $KEYCHAIN_PATH security list-keychain -d user -s $KEYCHAIN_PATH - name: Build app run: ./contrib/osx/make_osx $CODESIGN_CERT