From 36b1c4a19a3ee664c631e499b5c52b0759bdcdec Mon Sep 17 00:00:00 2001 From: bitmammoth Date: Mon, 29 Jul 2019 11:34:49 -0700 Subject: [PATCH 1/3] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc3c116..9b8a492 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ This installation guide will be using Linux/Unix and Python 3.5 throughout. Python will need to be installed on your system. To install python-pip run ``` -sudo apt install python-pip3 +sudo apt install python3-pip ``` If it is already installed, it may need to be upgraded by running ``` -pip3 install —upgrade pip +pip3 install --upgrade pip ``` Once you have python-pip installed you will need to install the python dependencies by running From f60e417a5d2b93b3546733d96bde09719f494d25 Mon Sep 17 00:00:00 2001 From: bitmammoth Date: Wed, 21 Aug 2019 19:37:10 -0700 Subject: [PATCH 2/3] bfix --- README.md | 4 +- api.py | 44 +++++++++++++++--- config.json | 8 +++- data/debug.log | 0 lib/blockchain.py | 4 +- lib/db.py | 15 ++++++- lib/memechain.py | 33 +++++++++----- logger.py | 2 + sync.py | 111 ++++++++++++++++++++++++++++++++++------------ 9 files changed, 166 insertions(+), 55 deletions(-) mode change 100644 => 100755 data/debug.log diff --git a/README.md b/README.md index 9b8a492..fc3c116 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ This installation guide will be using Linux/Unix and Python 3.5 throughout. Python will need to be installed on your system. To install python-pip run ``` -sudo apt install python3-pip +sudo apt install python-pip3 ``` If it is already installed, it may need to be upgraded by running ``` -pip3 install --upgrade pip +pip3 install —upgrade pip ``` Once you have python-pip installed you will need to install the python dependencies by running diff --git a/api.py b/api.py index fb9b55c..c4bcdac 100644 --- a/api.py +++ b/api.py @@ -6,7 +6,10 @@ import re import falcon import pickle as pickle - +import sys +import magic +import psutil +import time from lib.ipfs import IPFSTools from lib.db import MemeChainDB from lib.memechain import MemeTx, Validate @@ -23,6 +26,18 @@ # Memechain allowed content types ALLOWED_IMAGE_TYPES = ('image/gif', 'image/jpeg', 'image/png') + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + def validate_image_type(req, resp, resource, params): if req.content_type not in ALLOWED_IMAGE_TYPES: logger.error('COMMAND %s Failed %s: %s' @@ -215,6 +230,11 @@ def on_post(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) + while check_running(): + time.sleep(3) + logger.info('Waiting for the synchronization process to complete add_meme') + + # Generate random placeholder img name img_placeholder_name = str(random.random()).split(".")[1] @@ -222,9 +242,11 @@ def on_post(self, req, resp): if ext == '.jpe': ext = '.jpg' + name = '{img_name}{ext}'.format(img_name=img_placeholder_name, ext=ext) image_path = os.path.join(config['DATA_DIR'], name) + # Write image to local storage with io.open(image_path, 'wb') as image_file: while True: @@ -234,6 +256,12 @@ def on_post(self, req, resp): image_file.write(chunk) + + if magic.from_file(image_path).lower().find(mimetypes.guess_extension(req.content_type)[1:]) < 0: + raise falcon.HTTPBadRequest( "Memechain error", + "Meme has not passed validation, file extension is wrong.") + + # Check file size meme_filesize = os.path.getsize(image_path) * 0.000001 # in MB @@ -248,18 +276,15 @@ def on_post(self, req, resp): # Add image to ipfs ipfs_id = IPFSTools().add_meme(image_path)['Hash'] - # Rename local img file to ipfs_id for easy reference new_name = '{img_name}{ext}'.format(img_name=ipfs_id, ext=ext) os.rename(image_path, os.path.join(config['DATA_DIR'], new_name)) - # Add to Kekcoin chain memetx = MemeTx(ipfs_id) prev_block_memes = db.get_prev_block_memes() if prev_block_memes: memetx.generate_hashlink(prev_block_memes) - try: Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes) @@ -270,13 +295,18 @@ def on_post(self, req, resp): logger.error('COMMAND %s Failed %s: %s' % (self.__class__.__name__, 'Memechain Error', - "Meme has not passed memechain validation, file extension not supported.")) + "Meme has not passed memechain validation, file extension not supported.%s") ) raise falcon.HTTPError(falcon.HTTP_400, "Memechain error", - "Meme has not passed validation, file extension not supported.") + "Meme has not passed validation, file extension not supported.%s" % e ) if memetx.is_meme_valid(): + + memetx.blockchain_write() + db.add_meme(**{"ipfs_id": ipfs_id, "hashlink": memetx.get_hashlink(), + "txid": memetx.get_txid(), "author": memetx.get_author(), "block": 0, "imgformat": ext[1:], + "status": "unconfirm"}) resp.status = falcon.HTTP_201 resp.set_header('Powered-By', 'Memechain') @@ -335,4 +365,4 @@ def on_post(self, req, resp): app.add_route('/api/getmemeimgbyhash/{ipfs_id}', get_meme_img_by_hash()) # Add meme command -app.add_route('/api/addmeme', add_meme()) \ No newline at end of file +app.add_route('/api/addmeme', add_meme()) diff --git a/config.json b/config.json index c89a43d..2d5b9fa 100644 --- a/config.json +++ b/config.json @@ -3,5 +3,9 @@ "RPC_USER" : "user", "RPC_PASS" : "pass", "RPC_PORT" : "13377", - "ALLOWED_IP_ADDRESSES" : [] -} \ No newline at end of file + "ALLOWED_IP_ADDRESSES" : [], + "ALLOWED_IMAGE_EXTENSIONS" : ["jpg","png","gif"], + "ENABLE_LOG_MEMTX_NOT_FOUND" : false, + "MULTIPLE_SYNC_RUNNING" : false, + "CHECK_FILES_ON_RUNNING" : true +} diff --git a/data/debug.log b/data/debug.log old mode 100644 new mode 100755 diff --git a/lib/blockchain.py b/lib/blockchain.py index 4f6f111..bd4bfbe 100644 --- a/lib/blockchain.py +++ b/lib/blockchain.py @@ -108,9 +108,9 @@ def create_raw_op_return_transaction(metadata): "This tool set does not currently support reading op_return data with less than 4 chars") input_tx = get_input() - + print(round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8)) init_raw_tx = rpc.createrawtransaction([{"txid": input_tx["txid"], "vout": input_tx["vout"]}], { - input_tx["address"]: TX_BURN_AMOUNT, rpc.getnewaddress(): round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8)}) + input_tx["address"]: round(float(input_tx["amount"]) - 1.1 * TX_BURN_AMOUNT, 8), rpc.getnewaddress(): TX_BURN_AMOUNT}) oldScriptPubKey = init_raw_tx[len(init_raw_tx) - 60:len(init_raw_tx) - 8] newScriptPubKey = b"6a" + hexlify(bytes(chr(len(metadata)), encoding='utf-8')) + hexlify(bytes(metadata, encoding='utf-8')) diff --git a/lib/db.py b/lib/db.py index 53c9c48..df6c765 100644 --- a/lib/db.py +++ b/lib/db.py @@ -43,9 +43,9 @@ class MemeChainDB(object): def __init__(self, db_path): self._db = TinyDB(db_path) - def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author): + def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author, status): self._db.insert({"ipfs_id": ipfs_id, "hashlink": hashlink, - "txid": txid, "block": block, "imgformat" : imgformat, "author" : author}) + "txid": txid, "block": block, "imgformat" : imgformat, "author" : author, "status": status}) def remove_meme(self, ipfs_id): self._db.remove(Query().ipfs_id == ipfs_id) @@ -53,6 +53,14 @@ def remove_meme(self, ipfs_id): def get_memechain_height(self): return len(self._db) + def update_meme(self, ipfs_id, block): + memes = self._db.search(Query().ipfs_id == ipfs_id) + print(memes) + for meme in memes: + meme["block"] = block + meme["status"] = "confirm" + self._db.write_back(memes) + def search_by_block(self, block): """ Get a meme entry using block number as the search parameter @@ -119,5 +127,8 @@ def get_prev_block_memes(self): def get_last_meme(self): return Index(self._db)[-1] + def get_all_memes(self): + return self._db.search(Query().ipfs_id.exists()) + def get_meme_height_by_ipfs_id(self, ipfs_id): return Index(self._db).return_index(self._db.get(Query().ipfs_id == ipfs_id)) + 1 diff --git a/lib/memechain.py b/lib/memechain.py index 2997434..d755c80 100644 --- a/lib/memechain.py +++ b/lib/memechain.py @@ -1,6 +1,6 @@ import os from hashlib import sha256 - +from logger import * from .blockchain import * from .ipfs import IPFSTools @@ -14,13 +14,13 @@ def __init__(self, MemeTX, db, ipfs_dir, prev_block_memes, sync = False, genesis if genesis == False: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir), self.is_valid_hash_link(MemeTX, prev_block_memes), - self.check_duplicate(db, MemeTX.get_ipfs_id())] + self.check_duplicate(db, MemeTX.get_ipfs_id()), + self.check_file_ext(MemeTX.get_ipfs_id())] else: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir)] if sync == True: self.is_valid.append(self.check_burn_amount(MemeTX)) - MemeTX.set_is_valid(self.is_valid) def is_valid_hash_link(self, MemeTX, prev_block_memes): @@ -33,6 +33,7 @@ def is_valid_hash_link(self, MemeTX, prev_block_memes): raw_str += ''.join(meme['ipfs_id'] for meme in prev_block_memes) hashlink = sha256(raw_str.encode('utf-8')).hexdigest()[:16] if hashlink != MemeTX.get_hashlink(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Hashlink %s." % hashlink)) return False else: return True @@ -52,6 +53,7 @@ def check_ipfs_existance(self, ipfs_id, ipfs_dir): # IPFS Tools should be instanciated. ipfs = IPFSTools() if not ipfs.get_meme(ipfs_id, ipfs_dir): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme does not exist on IPFS yet %s." % ipfs_id)) return False # Meme does not exist on IPFS yet else: return True # Meme already exists on global IPFS @@ -65,8 +67,8 @@ def check_duplicate(self, db, ipfs_id): ipfs_id - attribute of MemeTX """ meme = db.search_by_ipfs_id(ipfs_id) - - if meme: + if meme and meme["status"] == "confirm": + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme exist on db yet %s." % ipfs_id)) return False else: return True @@ -76,12 +78,20 @@ def check_burn_amount(self, MemeTX): Checks whether the correct amount of KEKs were burned for the MemeTx. """ burn_amount = get_tx_burn_amount(MemeTX.get_txid()) - if float(burn_amount) == TX_BURN_AMOUNT: return True else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct amount of KEKs were burned for the MemeTx %s." % MemeTX.get_ipfs_id())) return False + def check_file_ext(self, meme): + meme_filepath = IPFSTools().get_meme(meme, config['DATA_DIR']) + ext = meme_filepath.split(".")[-1] + if ext in config["ALLOWED_IMAGE_EXTENSIONS"]: + return True + else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct image file extension %s." % ext)) + return False class MemeTx(object): """ @@ -99,10 +109,12 @@ def set_is_valid(self, values): self._is_valid = values def is_meme_valid(self): - if False not in self._is_valid: - return True - else: - return False + + for num, value in enumerate(self._is_valid, start=1): + if not value: + return num + + return -1 def get_ipfs_id(self): return self.ipfs_id @@ -140,7 +152,6 @@ def generate_hashlink(self, prev_block_memes): def blockchain_write(self): metadata = self._identifier + self.command_bytes + self.ipfs_id + self.hashlink - rawtx, self.author = create_raw_op_return_transaction(metadata) signedtx = sign_raw_transaction(rawtx) self.txid = send_raw_transaction(signedtx) diff --git a/logger.py b/logger.py index 80384f5..48fe43a 100644 --- a/logger.py +++ b/logger.py @@ -13,3 +13,5 @@ hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) + + diff --git a/sync.py b/sync.py index 7a5bfa9..5aec5d8 100644 --- a/sync.py +++ b/sync.py @@ -2,11 +2,12 @@ import os import json import pickle as pickle - +import time from lib.db import MemeChainDB from lib.blockchain import * from lib.memechain import MemeTx, Validate from lib.ipfs import IPFSTools +import psutil # Load configuration file with open("config.json", "r") as f: @@ -78,7 +79,6 @@ def return_memetxs(self): def sync_block(db, block): parser = MemechainParser(block) parser.collect_memetxs() - memetxs = parser.return_memetxs() if memetxs: @@ -88,36 +88,72 @@ def sync_block(db, block): memetx = MemeTx(meme['ipfs_id']) memetx.generate_hashlink(prev_block_memes) memetx.txid = meme['txid'] - Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes, sync=True) - - if memetx.is_meme_valid(): + valid_state = memetx.is_meme_valid() + if valid_state == -1: meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) ext = meme_filepath.split(".")[-1] - if ext in ["jpg", "png", "gif"]: + if db.search_by_ipfs_id(meme['ipfs_id']): + db.update_meme(meme['ipfs_id'], block) + logger.info('COMMAND %s Success %s: %s' % ( + 'Sync', 'Memechain', "Meme %s update in database." % meme['ipfs_id'])) + else: db.add_meme(**{"ipfs_id": meme['ipfs_id'], "hashlink": meme['hashlink'], - "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext}) - + "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext, "status": "confirm"}) logger.info('COMMAND %s Success %s: %s' % ('Sync', 'Memechain', "Meme %s added to database." % meme['ipfs_id'])) - - else: - # Delete invalid Meme - os.remove(meme_filepath) - - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Meme image extension %s." % ext)) - - else: + elif valid_state != 3: + meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) + os.remove(meme_filepath) logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid MemeTx %s." % meme['ipfs_id'])) else: - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + if config["ENABLE_LOG_MEMTX_NOT_FOUND"]: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + +def check_files_status(db): + for file in os.listdir(config['DATA_DIR']): + if file.endswith(tuple(config["ALLOWED_IMAGE_EXTENSIONS"])): + if db.search_by_ipfs_id(file.split(".")[0]) is None: + os.remove(os.path.join(config['DATA_DIR'], file)) + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Remove file from data folder (not exist in db) %s." % file)) + + for meme in db.get_all_memes(): + if not os.path.isfile(os.path.join(config['DATA_DIR'], meme["ipfs_id"] + "." + meme["imgformat"])): + logger.info('COMMAND %s Failed %s: %s' % ( + 'Sync', 'Memechain', "File not found in data folder (exist in db) %s." % meme["ipfs_id"])) + ipfs = IPFSTools() + ipfs.get_meme(meme["ipfs_id"], config['DATA_DIR']) + logger.info('COMMAND %s Info %s: %s' % ('Sync', 'Memechain', "Try downloading missing file with id %s." % meme["ipfs_id"])) + + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + + if __name__ == '__main__': # Load database db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) - + + + + if not config["MULTIPLE_SYNC_RUNNING"] and check_running(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Sync process already running.Shutdown current sync")) + sys.exit(1) + + if config["CHECK_FILES_ON_RUNNING"]: + check_files_status(db) + # Check blockheight block_height = get_block_height() @@ -128,27 +164,34 @@ def sync_block(db, block): memetx = MemeTx(genesis_meme.genesis_ipfs_id) memetx.generate_genesis_hashlink() memetx.txid = genesis_meme.genesis_txid - + Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=[], sync=True, genesis=True) - # Add genesis meme to database + + db.add_meme(**{"ipfs_id": genesis_meme.get_ipfs_id(), "hashlink": genesis_meme.get_hashlink(), - "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format}) + "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format, "status": "confirm"}) # Sync loop if genesis_meme.genesis_kekcoin_block < block_height: - for block in range(genesis_meme.genesis_kekcoin_block + 1, block_height + 1): + block = genesis_meme.genesis_kekcoin_block + 1 + max_errors = 0 + while block < block_height + 1: try: sync_block(db, block) except IOError as e: - logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) - - # Dump current sync height into a pickle - pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + block = block + 1 else: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Blockchain Error', "Kekcoin blockchain syncing...")) @@ -162,15 +205,25 @@ def sync_block(db, block): synced_height = last_meme['block'] # Sync loop - if synced_height < block_height: - for block in range(synced_height + 1, block_height + 1): + + if synced_height-10 < block_height: + block = synced_height-10 + 1 + max_errors = 0 + while block < block_height + 1: try: sync_block(db, block) except IOError as e: - logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + block = block + 1 # Dump current sync height into a pickle pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) From 39772af6e6126ec8e457d75723bce8184e5f4efa Mon Sep 17 00:00:00 2001 From: bitmammoth Date: Wed, 21 Aug 2019 19:41:10 -0700 Subject: [PATCH 3/3] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc3c116..9b8a492 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ This installation guide will be using Linux/Unix and Python 3.5 throughout. Python will need to be installed on your system. To install python-pip run ``` -sudo apt install python-pip3 +sudo apt install python3-pip ``` If it is already installed, it may need to be upgraded by running ``` -pip3 install —upgrade pip +pip3 install --upgrade pip ``` Once you have python-pip installed you will need to install the python dependencies by running