From 865ffa56481781e55dbd8ff6ac4b4ed1f70b3a12 Mon Sep 17 00:00:00 2001 From: roman efimov Date: Tue, 3 Sep 2019 18:13:59 +0400 Subject: [PATCH 1/7] Add meme to local db and check if sync already running and check image extension (mime type) --- api.py | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/api.py b/api.py index fb9b55c..c4bcdac 100644 --- a/api.py +++ b/api.py @@ -6,7 +6,10 @@ import re import falcon import pickle as pickle - +import sys +import magic +import psutil +import time from lib.ipfs import IPFSTools from lib.db import MemeChainDB from lib.memechain import MemeTx, Validate @@ -23,6 +26,18 @@ # Memechain allowed content types ALLOWED_IMAGE_TYPES = ('image/gif', 'image/jpeg', 'image/png') + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + def validate_image_type(req, resp, resource, params): if req.content_type not in ALLOWED_IMAGE_TYPES: logger.error('COMMAND %s Failed %s: %s' @@ -215,6 +230,11 @@ def on_post(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) + while check_running(): + time.sleep(3) + logger.info('Waiting for the synchronization process to complete add_meme') + + # Generate random placeholder img name img_placeholder_name = str(random.random()).split(".")[1] @@ -222,9 +242,11 @@ def on_post(self, req, resp): if ext == '.jpe': ext = '.jpg' + name = '{img_name}{ext}'.format(img_name=img_placeholder_name, ext=ext) image_path = os.path.join(config['DATA_DIR'], name) + # Write image to local storage with io.open(image_path, 'wb') as image_file: while True: @@ -234,6 +256,12 @@ def on_post(self, req, resp): image_file.write(chunk) + + if magic.from_file(image_path).lower().find(mimetypes.guess_extension(req.content_type)[1:]) < 0: + raise falcon.HTTPBadRequest( "Memechain error", + "Meme has not passed validation, file extension is wrong.") + + # Check file size meme_filesize = os.path.getsize(image_path) * 0.000001 # in MB @@ -248,18 +276,15 @@ def on_post(self, req, resp): # Add image to ipfs ipfs_id = IPFSTools().add_meme(image_path)['Hash'] - # Rename local img file to ipfs_id for easy reference new_name = '{img_name}{ext}'.format(img_name=ipfs_id, ext=ext) os.rename(image_path, os.path.join(config['DATA_DIR'], new_name)) - # Add to Kekcoin chain memetx = MemeTx(ipfs_id) prev_block_memes = db.get_prev_block_memes() if prev_block_memes: memetx.generate_hashlink(prev_block_memes) - try: Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes) @@ -270,13 +295,18 @@ def on_post(self, req, resp): logger.error('COMMAND %s Failed %s: %s' % (self.__class__.__name__, 'Memechain Error', - "Meme has not passed memechain validation, file extension not supported.")) + "Meme has not passed memechain validation, file extension not supported.%s") ) raise falcon.HTTPError(falcon.HTTP_400, "Memechain error", - "Meme has not passed validation, file extension not supported.") + "Meme has not passed validation, file extension not supported.%s" % e ) if memetx.is_meme_valid(): + + memetx.blockchain_write() + db.add_meme(**{"ipfs_id": ipfs_id, "hashlink": memetx.get_hashlink(), + "txid": memetx.get_txid(), "author": memetx.get_author(), "block": 0, "imgformat": ext[1:], + "status": "unconfirm"}) resp.status = falcon.HTTP_201 resp.set_header('Powered-By', 'Memechain') @@ -335,4 +365,4 @@ def on_post(self, req, resp): app.add_route('/api/getmemeimgbyhash/{ipfs_id}', get_meme_img_by_hash()) # Add meme command -app.add_route('/api/addmeme', add_meme()) \ No newline at end of file +app.add_route('/api/addmeme', add_meme()) From 804e8145caf6d35054e4c4f81a27c9dc49bbc231 Mon Sep 17 00:00:00 2001 From: roman efimov Date: Tue, 3 Sep 2019 18:23:59 +0400 Subject: [PATCH 2/7] image extensions now in config. Check image extension in memchain not in sync. Add new config options. Add check if sync already running. Add updating mem status (block number) --- config.json | 10 +++-- lib/memechain.py | 33 +++++++++----- sync.py | 111 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 111 insertions(+), 43 deletions(-) diff --git a/config.json b/config.json index c89a43d..7dd55be 100644 --- a/config.json +++ b/config.json @@ -1,7 +1,11 @@ { - "DATA_DIR" : "/root/memechain-api/data", + "DATA_DIR" : "/home/parallels/memechain-api/data", "RPC_USER" : "user", "RPC_PASS" : "pass", "RPC_PORT" : "13377", - "ALLOWED_IP_ADDRESSES" : [] -} \ No newline at end of file + "ALLOWED_IP_ADDRESSES" : [], + "ALLOWED_IMAGE_EXTENSIONS" : ["jpg","png","gif"], + "ENABLE_LOG_MEMTX_NOT_FOUND" : false, + "MULTIPLE_SYNC_RUNNING" : false, + "CHECK_FILES_ON_RUNNING" : true +} diff --git a/lib/memechain.py b/lib/memechain.py index 2997434..d755c80 100644 --- a/lib/memechain.py +++ b/lib/memechain.py @@ -1,6 +1,6 @@ import os from hashlib import sha256 - +from logger import * from .blockchain import * from .ipfs import IPFSTools @@ -14,13 +14,13 @@ def __init__(self, MemeTX, db, ipfs_dir, prev_block_memes, sync = False, genesis if genesis == False: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir), self.is_valid_hash_link(MemeTX, prev_block_memes), - self.check_duplicate(db, MemeTX.get_ipfs_id())] + self.check_duplicate(db, MemeTX.get_ipfs_id()), + self.check_file_ext(MemeTX.get_ipfs_id())] else: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir)] if sync == True: self.is_valid.append(self.check_burn_amount(MemeTX)) - MemeTX.set_is_valid(self.is_valid) def is_valid_hash_link(self, MemeTX, prev_block_memes): @@ -33,6 +33,7 @@ def is_valid_hash_link(self, MemeTX, prev_block_memes): raw_str += ''.join(meme['ipfs_id'] for meme in prev_block_memes) hashlink = sha256(raw_str.encode('utf-8')).hexdigest()[:16] if hashlink != MemeTX.get_hashlink(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Hashlink %s." % hashlink)) return False else: return True @@ -52,6 +53,7 @@ def check_ipfs_existance(self, ipfs_id, ipfs_dir): # IPFS Tools should be instanciated. ipfs = IPFSTools() if not ipfs.get_meme(ipfs_id, ipfs_dir): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme does not exist on IPFS yet %s." % ipfs_id)) return False # Meme does not exist on IPFS yet else: return True # Meme already exists on global IPFS @@ -65,8 +67,8 @@ def check_duplicate(self, db, ipfs_id): ipfs_id - attribute of MemeTX """ meme = db.search_by_ipfs_id(ipfs_id) - - if meme: + if meme and meme["status"] == "confirm": + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme exist on db yet %s." % ipfs_id)) return False else: return True @@ -76,12 +78,20 @@ def check_burn_amount(self, MemeTX): Checks whether the correct amount of KEKs were burned for the MemeTx. """ burn_amount = get_tx_burn_amount(MemeTX.get_txid()) - if float(burn_amount) == TX_BURN_AMOUNT: return True else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct amount of KEKs were burned for the MemeTx %s." % MemeTX.get_ipfs_id())) return False + def check_file_ext(self, meme): + meme_filepath = IPFSTools().get_meme(meme, config['DATA_DIR']) + ext = meme_filepath.split(".")[-1] + if ext in config["ALLOWED_IMAGE_EXTENSIONS"]: + return True + else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct image file extension %s." % ext)) + return False class MemeTx(object): """ @@ -99,10 +109,12 @@ def set_is_valid(self, values): self._is_valid = values def is_meme_valid(self): - if False not in self._is_valid: - return True - else: - return False + + for num, value in enumerate(self._is_valid, start=1): + if not value: + return num + + return -1 def get_ipfs_id(self): return self.ipfs_id @@ -140,7 +152,6 @@ def generate_hashlink(self, prev_block_memes): def blockchain_write(self): metadata = self._identifier + self.command_bytes + self.ipfs_id + self.hashlink - rawtx, self.author = create_raw_op_return_transaction(metadata) signedtx = sign_raw_transaction(rawtx) self.txid = send_raw_transaction(signedtx) diff --git a/sync.py b/sync.py index 7a5bfa9..5aec5d8 100644 --- a/sync.py +++ b/sync.py @@ -2,11 +2,12 @@ import os import json import pickle as pickle - +import time from lib.db import MemeChainDB from lib.blockchain import * from lib.memechain import MemeTx, Validate from lib.ipfs import IPFSTools +import psutil # Load configuration file with open("config.json", "r") as f: @@ -78,7 +79,6 @@ def return_memetxs(self): def sync_block(db, block): parser = MemechainParser(block) parser.collect_memetxs() - memetxs = parser.return_memetxs() if memetxs: @@ -88,36 +88,72 @@ def sync_block(db, block): memetx = MemeTx(meme['ipfs_id']) memetx.generate_hashlink(prev_block_memes) memetx.txid = meme['txid'] - Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes, sync=True) - - if memetx.is_meme_valid(): + valid_state = memetx.is_meme_valid() + if valid_state == -1: meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) ext = meme_filepath.split(".")[-1] - if ext in ["jpg", "png", "gif"]: + if db.search_by_ipfs_id(meme['ipfs_id']): + db.update_meme(meme['ipfs_id'], block) + logger.info('COMMAND %s Success %s: %s' % ( + 'Sync', 'Memechain', "Meme %s update in database." % meme['ipfs_id'])) + else: db.add_meme(**{"ipfs_id": meme['ipfs_id'], "hashlink": meme['hashlink'], - "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext}) - + "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext, "status": "confirm"}) logger.info('COMMAND %s Success %s: %s' % ('Sync', 'Memechain', "Meme %s added to database." % meme['ipfs_id'])) - - else: - # Delete invalid Meme - os.remove(meme_filepath) - - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Meme image extension %s." % ext)) - - else: + elif valid_state != 3: + meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) + os.remove(meme_filepath) logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid MemeTx %s." % meme['ipfs_id'])) else: - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + if config["ENABLE_LOG_MEMTX_NOT_FOUND"]: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + +def check_files_status(db): + for file in os.listdir(config['DATA_DIR']): + if file.endswith(tuple(config["ALLOWED_IMAGE_EXTENSIONS"])): + if db.search_by_ipfs_id(file.split(".")[0]) is None: + os.remove(os.path.join(config['DATA_DIR'], file)) + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Remove file from data folder (not exist in db) %s." % file)) + + for meme in db.get_all_memes(): + if not os.path.isfile(os.path.join(config['DATA_DIR'], meme["ipfs_id"] + "." + meme["imgformat"])): + logger.info('COMMAND %s Failed %s: %s' % ( + 'Sync', 'Memechain', "File not found in data folder (exist in db) %s." % meme["ipfs_id"])) + ipfs = IPFSTools() + ipfs.get_meme(meme["ipfs_id"], config['DATA_DIR']) + logger.info('COMMAND %s Info %s: %s' % ('Sync', 'Memechain', "Try downloading missing file with id %s." % meme["ipfs_id"])) + + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + + if __name__ == '__main__': # Load database db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) - + + + + if not config["MULTIPLE_SYNC_RUNNING"] and check_running(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Sync process already running.Shutdown current sync")) + sys.exit(1) + + if config["CHECK_FILES_ON_RUNNING"]: + check_files_status(db) + # Check blockheight block_height = get_block_height() @@ -128,27 +164,34 @@ def sync_block(db, block): memetx = MemeTx(genesis_meme.genesis_ipfs_id) memetx.generate_genesis_hashlink() memetx.txid = genesis_meme.genesis_txid - + Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=[], sync=True, genesis=True) - # Add genesis meme to database + + db.add_meme(**{"ipfs_id": genesis_meme.get_ipfs_id(), "hashlink": genesis_meme.get_hashlink(), - "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format}) + "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format, "status": "confirm"}) # Sync loop if genesis_meme.genesis_kekcoin_block < block_height: - for block in range(genesis_meme.genesis_kekcoin_block + 1, block_height + 1): + block = genesis_meme.genesis_kekcoin_block + 1 + max_errors = 0 + while block < block_height + 1: try: sync_block(db, block) except IOError as e: - logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) - - # Dump current sync height into a pickle - pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + block = block + 1 else: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Blockchain Error', "Kekcoin blockchain syncing...")) @@ -162,15 +205,25 @@ def sync_block(db, block): synced_height = last_meme['block'] # Sync loop - if synced_height < block_height: - for block in range(synced_height + 1, block_height + 1): + + if synced_height-10 < block_height: + block = synced_height-10 + 1 + max_errors = 0 + while block < block_height + 1: try: sync_block(db, block) except IOError as e: - logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + block = block + 1 # Dump current sync height into a pickle pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) From f5b213c048550c48807f5321ffbd1edea5a5e853 Mon Sep 17 00:00:00 2001 From: roman efimov Date: Tue, 3 Sep 2019 18:27:11 +0400 Subject: [PATCH 3/7] update append image in db method (add status). Add get all memes for check images exists. --- lib/db.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/db.py b/lib/db.py index 53c9c48..df6c765 100644 --- a/lib/db.py +++ b/lib/db.py @@ -43,9 +43,9 @@ class MemeChainDB(object): def __init__(self, db_path): self._db = TinyDB(db_path) - def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author): + def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author, status): self._db.insert({"ipfs_id": ipfs_id, "hashlink": hashlink, - "txid": txid, "block": block, "imgformat" : imgformat, "author" : author}) + "txid": txid, "block": block, "imgformat" : imgformat, "author" : author, "status": status}) def remove_meme(self, ipfs_id): self._db.remove(Query().ipfs_id == ipfs_id) @@ -53,6 +53,14 @@ def remove_meme(self, ipfs_id): def get_memechain_height(self): return len(self._db) + def update_meme(self, ipfs_id, block): + memes = self._db.search(Query().ipfs_id == ipfs_id) + print(memes) + for meme in memes: + meme["block"] = block + meme["status"] = "confirm" + self._db.write_back(memes) + def search_by_block(self, block): """ Get a meme entry using block number as the search parameter @@ -119,5 +127,8 @@ def get_prev_block_memes(self): def get_last_meme(self): return Index(self._db)[-1] + def get_all_memes(self): + return self._db.search(Query().ipfs_id.exists()) + def get_meme_height_by_ipfs_id(self, ipfs_id): return Index(self._db).return_index(self._db.get(Query().ipfs_id == ipfs_id)) + 1 From 5ae19a804a64666a3d115cfe64fa7edb030b88ff Mon Sep 17 00:00:00 2001 From: roman efimov Date: Tue, 3 Sep 2019 18:48:24 +0400 Subject: [PATCH 4/7] fix burn_amount --- lib/blockchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.py b/lib/blockchain.py index 4f6f111..bd4bfbe 100644 --- a/lib/blockchain.py +++ b/lib/blockchain.py @@ -108,9 +108,9 @@ def create_raw_op_return_transaction(metadata): "This tool set does not currently support reading op_return data with less than 4 chars") input_tx = get_input() - + print(round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8)) init_raw_tx = rpc.createrawtransaction([{"txid": input_tx["txid"], "vout": input_tx["vout"]}], { - input_tx["address"]: TX_BURN_AMOUNT, rpc.getnewaddress(): round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8)}) + input_tx["address"]: round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8), rpc.getnewaddress(): TX_BURN_AMOUNT}) oldScriptPubKey = init_raw_tx[len(init_raw_tx) - 60:len(init_raw_tx) - 8] newScriptPubKey = b"6a" + hexlify(bytes(chr(len(metadata)), encoding='utf-8')) + hexlify(bytes(metadata, encoding='utf-8')) From 2eb779e8ef5de48868e254595d10d7324cd46b7a Mon Sep 17 00:00:00 2001 From: roman efimov Date: Thu, 17 Oct 2019 19:52:16 +0400 Subject: [PATCH 5/7] fix burm from random wallet / remove debug print / add rpc_ip to config addmeme method example: http POST localhost:1337/api/addmeme addr==KMMqGN1AQ6fzrjy6t3UTnu6L5FrwiNLyou Content-Type:image/png < /media/psf/Home/Pictures/py7.png --- api.py | 4 ++-- config.json | 1 + lib/blockchain.py | 37 +++++++++++++------------------------ lib/memechain.py | 5 +++-- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/api.py b/api.py index b13515f..6ade937 100644 --- a/api.py +++ b/api.py @@ -304,7 +304,7 @@ def on_post(self, req, resp): new_name = '{img_name}{ext}'.format(img_name=ipfs_id, ext=ext) os.rename(image_path, os.path.join(config['DATA_DIR'], new_name)) # Add to Kekcoin chain - memetx = MemeTx(ipfs_id) + memetx = MemeTx(ipfs_id, req.params['addr']) prev_block_memes = db.get_prev_block_memes() if prev_block_memes: @@ -350,7 +350,7 @@ def on_post(self, req, resp): "Meme has not passed validation: ") else: # Genesis block logic - memetx = MemeTx(ipfs_id) + memetx = MemeTx(ipfs_id, req.params['addr']) memetx.generate_genesis_hashlink() memetx.blockchain_write() diff --git a/config.json b/config.json index 7dd55be..5de211a 100644 --- a/config.json +++ b/config.json @@ -2,6 +2,7 @@ "DATA_DIR" : "/home/parallels/memechain-api/data", "RPC_USER" : "user", "RPC_PASS" : "pass", + "RPC_IP" : "127.0.0.1", "RPC_PORT" : "13377", "ALLOWED_IP_ADDRESSES" : [], "ALLOWED_IMAGE_EXTENSIONS" : ["jpg","png","gif"], diff --git a/lib/blockchain.py b/lib/blockchain.py index bd4bfbe..2b252d9 100644 --- a/lib/blockchain.py +++ b/lib/blockchain.py @@ -29,8 +29,7 @@ def get_blockchain_info(): Returns: getinfo output (dict) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.getinfo() @@ -41,8 +40,7 @@ def get_block_height(): Returns: Block height (int) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.getblockcount() @@ -57,8 +55,7 @@ def get_block_txs(height): Returns: List of transaction ids (array) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) block_hash = rpc.getblockhash(height) block = rpc.getblock(block_hash) @@ -66,27 +63,26 @@ def get_block_txs(height): return block['tx'] -def get_input(): +def get_input(addr): """ Method used to get unspent inputs Returns: Transaction object (dict) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) unspent = rpc.listunspent() for tx in unspent: - if float(tx["amount"]) > 0.01: + if float(tx["amount"]) > 0.01 and addr == tx['address']: return tx else: raise Exception( "No valid inputs, inputs must be greater than 0.001 KEK") -def create_raw_op_return_transaction(metadata): +def create_raw_op_return_transaction(metadata,addr): """ Method used to create a transaction with embedded data through OP_RETURN @@ -97,8 +93,7 @@ def create_raw_op_return_transaction(metadata): Raw transaction (hex) Author address (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) if sys.getsizeof(metadata) > MAX_OP_RETURN_BYTES: raise Exception("Metadata size is over MAX_OP_RETURN_BYTES") @@ -107,8 +102,7 @@ def create_raw_op_return_transaction(metadata): raise Exception( "This tool set does not currently support reading op_return data with less than 4 chars") - input_tx = get_input() - print(round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8)) + input_tx = get_input(addr) init_raw_tx = rpc.createrawtransaction([{"txid": input_tx["txid"], "vout": input_tx["vout"]}], { input_tx["address"]: round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8), rpc.getnewaddress(): TX_BURN_AMOUNT}) @@ -121,7 +115,6 @@ def create_raw_op_return_transaction(metadata): op_return_tx = init_raw_tx.replace(oldScriptPubKey, newScriptPubKey.decode('ascii')) - print(rpc.decoderawtransaction(op_return_tx)['vout']) return op_return_tx, input_tx["address"] @@ -136,8 +129,7 @@ def sign_raw_transaction(tx): Returns: Signed raw transaction (hex) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.signrawtransaction(tx)["hex"] @@ -152,8 +144,7 @@ def send_raw_transaction(tx_hex): Returns: Transaction id (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.sendrawtransaction(tx_hex) @@ -169,8 +160,7 @@ def get_op_return_data(txid): Embedded metadata (str) Author address (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) raw_tx = rpc.getrawtransaction(txid) tx_data = rpc.decoderawtransaction(raw_tx) @@ -198,8 +188,7 @@ def get_tx_burn_amount(txid): Returns: Sum of input values, i.e. burn amount (float) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) raw_tx = rpc.getrawtransaction(txid) tx_data = rpc.decoderawtransaction(raw_tx) diff --git a/lib/memechain.py b/lib/memechain.py index d755c80..79d8e3e 100644 --- a/lib/memechain.py +++ b/lib/memechain.py @@ -98,12 +98,13 @@ class MemeTx(object): MemeChain TX object. Used to construct a MemeChainTX. """ - def __init__(self, ipfs_id): + def __init__(self, ipfs_id, addr): # Identifier is first 4 letters of the SHA256 hash of KEK self._identifier = '3ae4' self.command_bytes = '00' self.ipfs_id = ipfs_id self._is_valid = False + self.addr = addr def set_is_valid(self, values): self._is_valid = values @@ -152,6 +153,6 @@ def generate_hashlink(self, prev_block_memes): def blockchain_write(self): metadata = self._identifier + self.command_bytes + self.ipfs_id + self.hashlink - rawtx, self.author = create_raw_op_return_transaction(metadata) + rawtx, self.author = create_raw_op_return_transaction(metadata, self.addr) signedtx = sign_raw_transaction(rawtx) self.txid = send_raw_transaction(signedtx) From de97a1fed67fcd22a2c10aa334733bd5465fad5b Mon Sep 17 00:00:00 2001 From: roman efimov Date: Mon, 11 Nov 2019 16:17:23 +0400 Subject: [PATCH 6/7] fix add address param check fix MemeTx delegete now last block dump to sync.p before sync process (fix if ipfs timeout) Need new ipfs meme getter method --- api.py | 12 ++++++++++++ lib/memechain.py | 2 +- sync.py | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api.py b/api.py index 6ade937..b917bb9 100644 --- a/api.py +++ b/api.py @@ -55,6 +55,14 @@ def validate_ip_address(req, resp, resource, params): raise falcon.HTTPError(falcon.HTTP_401, 'Memechain Error', "IP address not allowed.") +def validate_address(req, resp, resource, params): + if req.params['addr'] == '': + logger.error('COMMAND %s Failed %s: %s' + % ('validate_address', 'Memechain Error', + "Address param must be not null.")) + raise falcon.HTTPError(falcon.HTTP_400, 'Memechain Error', + "Address param must be not null.") + class get_info(object): def on_get(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) @@ -242,10 +250,14 @@ class add_meme(object): @falcon.before(validate_ip_address) @falcon.before(validate_image_type) + @falcon.before(validate_address) + def on_post(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) + + while check_running(): time.sleep(3) logger.info('Waiting for the synchronization process to complete add_meme') diff --git a/lib/memechain.py b/lib/memechain.py index 79d8e3e..6bcd1dd 100644 --- a/lib/memechain.py +++ b/lib/memechain.py @@ -98,7 +98,7 @@ class MemeTx(object): MemeChain TX object. Used to construct a MemeChainTX. """ - def __init__(self, ipfs_id, addr): + def __init__(self, ipfs_id, addr=''): # Identifier is first 4 letters of the SHA256 hash of KEK self._identifier = '3ae4' self.command_bytes = '00' diff --git a/sync.py b/sync.py index 9a974ba..33369ca 100644 --- a/sync.py +++ b/sync.py @@ -164,7 +164,7 @@ def check_running(): # Load genesis meme genesis_meme = GenesisMeme() - memetx = MemeTx(genesis_meme.genesis_ipfs_id) + memetx = MemeTx(genesis_meme.genesis_ipfs_id,genesis_meme.genesis_author) memetx.generate_genesis_hashlink() memetx.txid = genesis_meme.genesis_txid @@ -233,6 +233,7 @@ def check_running(): max_errors = 0 while block < block_height + 1: try: + pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) sync_block(db, block) except InvalidMultihashError as e: From c2637d29c1adb4fd27d5c18bcc7e699d7728a84b Mon Sep 17 00:00:00 2001 From: roman efimov Date: Mon, 11 Nov 2019 16:53:08 +0400 Subject: [PATCH 7/7] fix merge conflict --- .idea/workspace.xml | 60 +++++++++++++++++++++++++++++++++++++++++++++ lib/blockchain.py | 31 +++++++++++++++-------- 2 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..f19f65c --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + 1573475351056 + + + + + + + + + \ No newline at end of file diff --git a/lib/blockchain.py b/lib/blockchain.py index 2b252d9..8b404ae 100644 --- a/lib/blockchain.py +++ b/lib/blockchain.py @@ -20,6 +20,7 @@ # OP_RETURN configuration TX_BURN_AMOUNT = 0.01 # Amount of KEKs to be burned in MemeTX +TX_FEE_RATE = 0.001 # Amount of KEKs to be paid as a miner fee MAX_OP_RETURN_BYTES = 500 def get_blockchain_info(): @@ -103,10 +104,15 @@ def create_raw_op_return_transaction(metadata,addr): "This tool set does not currently support reading op_return data with less than 4 chars") input_tx = get_input(addr) + init_raw_tx = rpc.createrawtransaction([{"txid": input_tx["txid"], "vout": input_tx["vout"]}], { - input_tx["address"]: round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8), rpc.getnewaddress(): TX_BURN_AMOUNT}) + rpc.getnewaddress(): TX_BURN_AMOUNT, + input_tx["address"]: round(float(input_tx["amount"]) - TX_BURN_AMOUNT - TX_FEE_RATE, 8)}) + + for vout in rpc.decoderawtransaction(init_raw_tx)["vout"]: + if float(vout["value"]) == TX_BURN_AMOUNT: + oldScriptPubKey = "19" + vout["scriptPubKey"]["hex"] - oldScriptPubKey = init_raw_tx[len(init_raw_tx) - 60:len(init_raw_tx) - 8] newScriptPubKey = b"6a" + hexlify(bytes(chr(len(metadata)), encoding='utf-8')) + hexlify(bytes(metadata, encoding='utf-8')) newScriptPubKey = hexlify(bytes(chr(len(unhexlify(newScriptPubKey))), encoding='utf-8')) + newScriptPubKey @@ -165,17 +171,22 @@ def get_op_return_data(txid): raw_tx = rpc.getrawtransaction(txid) tx_data = rpc.decoderawtransaction(raw_tx) - for data in tx_data["vout"]: - if data["scriptPubKey"]["asm"][:9] == "OP_RETURN": - op_return_data = str(unhexlify(data["scriptPubKey"]["asm"][10:]), encoding='utf-8') + if len(tx_data["vout"]) > 1: + if tx_data["vout"][0]["scriptPubKey"]["asm"][:9] == "OP_RETURN": + op_return_data = str(unhexlify(tx_data["vout"][0]["scriptPubKey"]["asm"][10:]), encoding='utf-8') + author = tx_data['vout'][1]['scriptPubKey']['addresses'][0] + + elif tx_data["vout"][1]["scriptPubKey"]["asm"][:9] == "OP_RETURN": + op_return_data = str(unhexlify(tx_data["vout"][1]["scriptPubKey"]["asm"][10:]), encoding='utf-8') + author = tx_data['vout'][0]['scriptPubKey']['addresses'][0] + else: op_return_data = None + author = None + else: + op_return_data = None + author = None - try: - author = tx_data['vout'][0]['scriptPubKey']['addresses'][0] - except KeyError as e: - author = None - return op_return_data, author def get_tx_burn_amount(txid):