From 2408d536fee2c4043fc3197d5d67c7df53c0eb9d Mon Sep 17 00:00:00 2001 From: Maksim Zubov Date: Thu, 2 May 2024 18:09:23 +0200 Subject: [PATCH] Initial commit Add scripts for uploading html-sites and setting domain records --- .env_template | 5 +++ .gitignore | 4 ++ README.md | 101 ++++++++++++++++++++++++++++++++++++++++++ abi/domain.abi.json | 58 ++++++++++++++++++++++++ abi/eversite.abi.json | 44 ++++++++++++++++++ config.py | 49 ++++++++++++++++++++ domain.py | 33 ++++++++++++++ eversite.py | 62 ++++++++++++++++++++++++++ requirements.txt | 5 +++ set_record.py | 85 +++++++++++++++++++++++++++++++++++ upload_site.py | 61 +++++++++++++++++++++++++ 11 files changed, 507 insertions(+) create mode 100644 .env_template create mode 100644 .gitignore create mode 100644 abi/domain.abi.json create mode 100644 abi/eversite.abi.json create mode 100644 config.py create mode 100644 domain.py create mode 100644 eversite.py create mode 100644 requirements.txt create mode 100644 set_record.py create mode 100644 upload_site.py diff --git a/.env_template b/.env_template new file mode 100644 index 0000000..3e5c62e --- /dev/null +++ b/.env_template @@ -0,0 +1,5 @@ +SEED_PHRASE="one two three four five six seven eight nine ten eleven twelve" +SITE_ADDRESS="0:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +DOMAIN_ADDRESS="0:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +GQL_TRANSPORT="https://mainnet.evercloud.dev/XXXXXX/graphql" +JRPC_TRANSPORT="https://jrpc.everwallet.net" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5630abb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +__pycache__/ +.env +sites/ \ No newline at end of file diff --git a/README.md b/README.md index 8b13789..5cd2bf5 100644 --- a/README.md +++ b/README.md @@ -1 +1,102 @@ +# Scripts for uploading html-sites to blockchain +It uses [.ever domains](https://app.evername.io/) for storing sites. +Every .ever domain is an NFT-contract with a set of records. +There are two options for storing sites: +1. Store directly in domain NFT-contract (record 1004) +2. Store in separate contract and link it to domain (record 1005) + +Scripts are written in Python 3. + +## Install dependencies + +```shell +python3 -m pip install -r requirements.txt +``` + +## Prepare the environment + +`.env` is used to configure these scripts. +You should create it like [env_template](.env_template). +Fill in the fields with your data. + +| Field | Required | Description | +|------------------|----------|------------------------------------------------------| +| `SEED_PHRASE` | **yes** | Seed phrase of the wallet that pays for transactions | +| `SITE_ADDRESS` | **yes** | Address of the site contract | +| `DOMAIN_ADDRESS` | **yes** | Address of the .ever domain NFT | +| `GQL_TRANSPORT` | no | Address of GraphQL transport | +| `JRPC_TRANSPORT` | no | Address of JSON-RPC transport | + +Take in mind that the account of the provided seed phrase has to be the owner of the domain and deployed site. + +### Transport + +Recommended way is to use GraphQL transport. +To get the right url of transport you should visit [everclod](https://www.evercloud.dev/), register and create a new project. +Then you will get the url of transport. + +You have to specify at least one transport in the environment, so either `GQL_TRANSPORT` or `JRPC_TRANSPORT` should be set. + +## Store site directly + +This way is strongly limited to store **only 889 bytes**! Use it to store tiny sites or html-formatted text. +Script tries to minify your html before storing, but it does not guarantee that it will fit. +You could use `set_record.py` script for this with the `1004` record key. + +```shell +python3 -m set_record -r 1004 sites/my_site.html +``` + +*Tip* You could store your html in `sites` folder as it is added to `.gitignore`. + +## Upload site to special contract + +### 1. Deploy site contract + +You could deploy [site contract from special repository](https://github.com/chumschat/eversite-contract). +You will get the contract address and use it in the next steps. + +### 2. Upload site + +After deploying the site contract you could upload html to it. +You could use `upload_site.py` script for this. +This script also tries to minify your html. + +```shell +python3 -m upload_site sites/my_site.html +``` + +*Tip* You could store your html in `sites` folder as it is added to `.gitignore`. + +### 3. Link domain to site + +After uploading the site you should link it to the domain. +You could use `set_record.py` script for this with the `1005` record key (default). + +```shell +python3 -m set_record -r 1005 +``` + +## Troubleshooting + +### RuntimeError: Message expired + +If you can't upload a site with constantly error `RuntimeError: Message expired`, +and your transaction is not accepted, that often means that the contract hasn't enough gas on its balance. + +Script `upload_site.py` uses external messages to send data to the contract. +It is the cheapest way to send *large* amount of data to the contract. +Such messages come without gas, so it is consumed from the site contract. +Script divides html into chunks of 60 000 symbols, you could see it in the logs. +Each of such chunk requires about 0.27-0.3 EVER for gas that should be on the contract balance. +I.e. for 5 chunks you should have about 1.5 EVER on the contract balance. + +### Shrink amount of chunks + +Currently, if you upload a site with multiple chunks, you wouldn't be able to upload it again with the smaller number of chinks. +Old high-indexed chunks will stay in the contract, and it's overall content will be corrupted. +You cannot do anything without it. You could only destroy old site using it's `destruct` method and redeploy new contract. +By doing this you could at least get the remaining gas back. + +Consider onchain sites as immutable. diff --git a/abi/domain.abi.json b/abi/domain.abi.json new file mode 100644 index 0000000..e2f99b1 --- /dev/null +++ b/abi/domain.abi.json @@ -0,0 +1,58 @@ +{ + "ABI version": 2, + "version": "2.2", + "header": ["pubkey", "time", "expire"], + "functions": [ + { + "name": "supportsInterface", + "inputs": [ + {"name":"answerId","type":"uint32"}, + {"name":"interfaceID","type":"uint32"} + ], + "outputs": [ + {"name":"support","type":"bool"} + ] + }, + { + "name": "getRecords", + "inputs": [ + {"name":"answerId","type":"uint32"} + ], + "outputs": [ + {"name":"records","type":"map(uint32,cell)"} + ] + }, + { + "name": "setRecord", + "inputs": [ + {"name":"key","type":"uint32"}, + {"name":"value","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "deleteRecord", + "inputs": [ + {"name":"key","type":"uint32"} + ], + "outputs": [ + ] + }, + { + "name": "getInfo", + "inputs": [ + {"name":"answerId","type":"uint32"} + ], + "outputs": [ + {"name":"id","type":"uint256"}, + {"name":"owner","type":"address"}, + {"name":"manager","type":"address"}, + {"name":"collection","type":"address"} + ] + } + ], + "data": [], + "events": [], + "fields": [] +} \ No newline at end of file diff --git a/abi/eversite.abi.json b/abi/eversite.abi.json new file mode 100644 index 0000000..8a390ec --- /dev/null +++ b/abi/eversite.abi.json @@ -0,0 +1,44 @@ +{ + "ABI version": 2, + "version": "2.3", + "header": ["pubkey", "time", "expire"], + "functions": [ + { + "name": "upload", + "inputs": [ + {"name":"part","type":"cell"}, + {"name":"index","type":"uint8"} + ], + "outputs": [ + ] + }, + { + "name": "getDetails", + "inputs": [ + ], + "outputs": [ + {"name":"value0","type":"map(uint8,cell)"} + ] + }, + { + "name": "owner", + "inputs": [ + ], + "outputs": [ + {"name":"owner","type":"uint256"} + ] + } + ], + "data": [ + {"key":1,"name":"nonce","type":"uint16"} + ], + "events": [ + ], + "fields": [ + {"name":"_pubkey","type":"uint256"}, + {"name":"_timestamp","type":"uint64"}, + {"name":"_constructorFlag","type":"bool"}, + {"name":"nonce","type":"uint16"}, + {"name":"content","type":"map(uint8,cell)"} + ] +} \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..1bbc453 --- /dev/null +++ b/config.py @@ -0,0 +1,49 @@ +import os + +from dotenv import load_dotenv +from pydantic import BaseModel, Field, model_validator +from typing_extensions import Self +import nekoton as nt + + +class Configuration(BaseModel): + seed_phrase: str = Field(..., min_length=1) + site_address: str = Field(..., min_length=1) + domain_address: str = Field(..., min_length=1) + gql_transport: str + jrpc_transport: str + + @model_validator(mode="after") + def check_model(self) -> Self: + if not nt.Address.validate(self.site_address): + raise ValueError("Invalid site address is provided in environment!") + if not nt.Address.validate(self.domain_address): + raise ValueError("Invalid domain address is provided in environment!") + return self + + async def get_transport(self) -> nt.Transport: + if self.gql_transport: + transport = nt.GqlTransport([self.gql_transport]) + await transport.check_connection() + return transport + elif self.jrpc_transport: + transport = nt.JrpcTransport(endpoint=self.jrpc_transport) + await transport.check_connection() + return transport + else: + raise ValueError("No valid transport is provided in environment!") + + def get_keypair(self) -> nt.KeyPair: + seed = nt.Bip39Seed(self.seed_phrase) + return seed.derive() + + +def load_config() -> Configuration: + load_dotenv() + return Configuration( + seed_phrase=os.getenv("SEED_PHRASE"), + site_address=os.getenv("SITE_ADDRESS"), + domain_address=os.getenv("DOMAIN_ADDRESS"), + gql_transport=os.getenv("GQL_TRANSPORT", ""), + jrpc_transport=os.getenv("JRPC_TRANSPORT", "") + ) diff --git a/domain.py b/domain.py new file mode 100644 index 0000000..c57c9f7 --- /dev/null +++ b/domain.py @@ -0,0 +1,33 @@ +import nekoton as nt +from typing import Optional + + +class Domain: + + def __init__(self, transport: nt.Transport, address: nt.Address): + self._address = address + self._transport = transport + + with open("abi/domain.abi.json") as f: + domain_abi_text = f.read() + domain_abi = nt.ContractAbi(domain_abi_text) + + self._getInfo = domain_abi.get_function("getInfo") + assert self._getInfo is not None + self._setRecord = domain_abi.get_function("setRecord") + assert self._setRecord is not None + + async def _get_account_state(self) -> Optional[nt.AccountState]: + return await self._transport.get_account_state(self._address) + + async def get_info(self) -> Optional[dict]: + state = await self._get_account_state() + + if state is None: + return None + else: + return self._getInfo.call(state, input={'answerId': 0}).output + + async def get_set_record_body(self, record_key: int, content: nt.Cell) -> nt.Cell: + input_params = {"key": record_key, "value": content} + return self._setRecord.encode_internal_input(input_params) diff --git a/eversite.py b/eversite.py new file mode 100644 index 0000000..a41a356 --- /dev/null +++ b/eversite.py @@ -0,0 +1,62 @@ +import nekoton as nt +from typing import Optional + + +class EverSite: + + def __init__(self, transport: nt.Transport, address: nt.Address, keypair: nt.KeyPair): + self._address = address + self._transport = transport + self._keypair = keypair + + with open("abi/eversite.abi.json") as f: + eversite_abi_text = f.read() + eversite_abi = nt.ContractAbi(eversite_abi_text) + + self._upload = eversite_abi.get_function("upload") + assert self._upload is not None + self._getDetails = eversite_abi.get_function("getDetails") + assert self._getDetails is not None + self._owner = eversite_abi.get_function("owner") + assert self._owner is not None + + async def _get_account_state(self) -> Optional[nt.AccountState]: + return await self._transport.get_account_state(self._address) + + async def get_details(self) -> Optional[nt.ExecutionOutput]: + state = await self._get_account_state() + + if state is None: + return None + else: + return self._getDetails.call(state, input={}) + + async def get_owner(self) -> Optional[nt.PublicKey]: + state = await self._get_account_state() + + if state is None: + return None + else: + public_key_int = self._owner.call(state, input={}).output["owner"] + return nt.PublicKey.from_int(public_key_int) + + async def build_upload(self, content: nt.Cell, index: int) -> nt.Cell: + input_params = {"index": index, "part": content} + return self._upload.encode_internal_input(input_params) + + async def send_chunk(self, content: nt.Cell, index: int) -> nt.Transaction: + signature_id = await self._transport.get_signature_id() + + external_message = self._upload.encode_external_message( + self._address, + input={ + "index": index, + "part": content + }, + public_key=self._keypair.public_key + ).sign(self._keypair, signature_id) + + tx = await self._transport.send_external_message(external_message) + if tx is None: + raise RuntimeError("Message expired") + return tx diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dad0e5d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +nekoton==0.1.18 +typing~=3.7.4.3 +htmlmin~=0.1.12 +python-dotenv==1.0.0 +pydantic~=2.1.1 \ No newline at end of file diff --git a/set_record.py b/set_record.py new file mode 100644 index 0000000..9ecf2ff --- /dev/null +++ b/set_record.py @@ -0,0 +1,85 @@ +import argparse +import asyncio +import logging +import os +import sys + +import htmlmin +import nekoton as nt + +from config import load_config +from domain import Domain + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("set_record") +logger.setLevel(logging.INFO) + +ONCHAIN_RECORD = 1004 +ONCHAIN_CONTRACT_RECORD = 1005 +SITE_SIZE_LIMIT = 889 + + +async def main(): + parser = argparse.ArgumentParser(description="Set .ever domain record to store html or site address") + parser.add_argument('-r', '--record', default=1005, help='type of record to store your site (1004 or 1005)') + parser.add_argument('html', nargs='?', help='path to html file for uploading') + args = parser.parse_args() + record = int(args.record) + file_path = args.html + if record not in [ONCHAIN_RECORD, ONCHAIN_CONTRACT_RECORD]: + sys.exit(f"Record type must be {ONCHAIN_RECORD} or {ONCHAIN_CONTRACT_RECORD}") + if record == ONCHAIN_RECORD: + if not file_path: + sys.exit(f"File path is required for {ONCHAIN_RECORD} record") + elif not os.path.exists(file_path): + sys.exit(f"File {file_path} not found!") + + config = load_config() + transport = await config.get_transport() + keypair = config.get_keypair() + ever_wallet = nt.contracts.EverWallet(transport, keypair) + address = ever_wallet.address + logger.info("Address %s is used as uploader", address) + + domain_address = nt.Address(config.domain_address) + domain = Domain(transport, domain_address) + + domain_info = await domain.get_info() + if domain_info["owner"] != address: + sys.exit(f"Uploader address {address} is not the owner of the domain {config.domain_address}") + + if record == ONCHAIN_CONTRACT_RECORD: + logger.info("Setting domain %s record %d to site address %s", + domain_address, record, config.site_address) + + cell_abi = [("value", nt.AbiAddress())] + cell_data = {"value": nt.Address(config.site_address)} + cell = nt.Cell.build(abi=cell_abi, value=cell_data) + payload = await domain.get_set_record_body(record, cell) + + tx = await ever_wallet.send(domain_address, nt.Tokens(1), payload, True) + logger.info("Domain %s record %d successfully set in transaction %s", + domain_address, record, tx.hash.hex()) + + if record == ONCHAIN_RECORD: + with open(file_path) as f: + site_source = f.read() + site_source_min = htmlmin.minify(site_source, remove_comments=True, remove_empty_space=True) + site_size = len(site_source_min.encode("utf-8")) + if site_size >= SITE_SIZE_LIMIT: + sys.exit(f"Html size is too big ({site_size} bytes), max size is 889 bytes") + + logger.info("Setting domain %s record %d to store html", + domain_address, record) + + cell_abi = [("value", nt.AbiString())] + cell_data = {"value": site_source_min} + cell = nt.Cell.build(abi=cell_abi, value=cell_data) + payload = await domain.get_set_record_body(record, cell) + + tx = await ever_wallet.send(domain_address, nt.Tokens(1), payload, True) + logger.info("Domain %s record %d successfully set in transaction %s", + domain_address, record, tx.hash.hex()) + + +asyncio.run(main()) diff --git a/upload_site.py b/upload_site.py new file mode 100644 index 0000000..98e82e2 --- /dev/null +++ b/upload_site.py @@ -0,0 +1,61 @@ +import argparse +import asyncio +import logging +import os +import sys + +import htmlmin +import nekoton as nt + +from config import load_config +from eversite import EverSite + +MESSAGE_SIZE = 60_000 + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("upload_site") +logger.setLevel(logging.INFO) + + +async def main(): + parser = argparse.ArgumentParser(description="Upload html site to eversite contract") + parser.add_argument('html', help='path to html file for uploading') + args = parser.parse_args() + file_path = args.html + if not os.path.exists(file_path): + sys.exit(f"File {file_path} not found") + + config = load_config() + transport = await config.get_transport() + keypair = config.get_keypair() + address = nt.contracts.EverWallet.compute_address(keypair.public_key) + site = EverSite(transport, nt.Address(config.site_address), keypair) + + logger.info("Address %s is used as uploader", address) + site_owner = await site.get_owner() + if site_owner != keypair.public_key: + sys.exit(f"Uploader address {address} is not the owner of the site {config.site_address}") + + logger.info("Reading and minifying html from file %s", file_path) + with open(file_path) as f: + site_source = f.read() + site_source_min = htmlmin.minify(site_source, remove_comments=True, remove_empty_space=True) + site_chunks = [] + for i in range(0, len(site_source_min), MESSAGE_SIZE): + site_chunks.append(site_source_min[i:i + MESSAGE_SIZE]) + + chunks = len(site_chunks) + logger.info("Minified html fits to %d chunks of %d symbols", chunks, MESSAGE_SIZE) + logger.info("Uploading %d chunks to site %s", chunks, config.site_address) + + for index, chunk in enumerate(site_chunks): + cell_abi = [("value", nt.AbiString())] + cell_data = {"value": chunk} + cell = nt.Cell.build(abi=cell_abi, value=cell_data) + tx = await site.send_chunk(cell, index) + logger.info("Uploaded %d/%d chunk in transaction %s", index+1, chunks, tx.hash.hex()) + + logger.info("Site uploaded successfully") + + +asyncio.run(main())