diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..f19f65c --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + 1573475351056 + + + + + + + + + \ No newline at end of file diff --git a/api.py b/api.py index 860e721..b917bb9 100644 --- a/api.py +++ b/api.py @@ -6,7 +6,10 @@ import re import falcon import pickle as pickle - +import sys +import magic +import psutil +import time from lib.ipfs import IPFSTools from lib.db import MemeChainDB from lib.memechain import MemeTx, Validate @@ -23,6 +26,18 @@ # Memechain allowed content types ALLOWED_IMAGE_TYPES = ('image/gif', 'image/jpeg', 'image/png') + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + def validate_image_type(req, resp, resource, params): if req.content_type not in ALLOWED_IMAGE_TYPES: logger.error('COMMAND %s Failed %s: %s' @@ -40,6 +55,14 @@ def validate_ip_address(req, resp, resource, params): raise falcon.HTTPError(falcon.HTTP_401, 'Memechain Error', "IP address not allowed.") +def validate_address(req, resp, resource, params): + if req.params['addr'] == '': + logger.error('COMMAND %s Failed %s: %s' + % ('validate_address', 'Memechain Error', + "Address param must be not null.")) + raise falcon.HTTPError(falcon.HTTP_400, 'Memechain Error', + "Address param must be not null.") + class get_info(object): def on_get(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) @@ -227,10 +250,19 @@ class add_meme(object): @falcon.before(validate_ip_address) @falcon.before(validate_image_type) + @falcon.before(validate_address) + def on_post(self, req, resp): logger.info('COMMAND %s Received' % self.__class__.__name__) db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) + + + while check_running(): + time.sleep(3) + logger.info('Waiting for the synchronization process to complete add_meme') + + # Generate random placeholder img name img_placeholder_name = str(random.random()).split(".")[1] @@ -238,9 +270,11 @@ def on_post(self, req, resp): if ext == '.jpe': ext = '.jpg' + name = '{img_name}{ext}'.format(img_name=img_placeholder_name, ext=ext) image_path = os.path.join(config['DATA_DIR'], name) + # Write image to local storage with io.open(image_path, 'wb') as image_file: while True: @@ -250,6 +284,12 @@ def on_post(self, req, resp): image_file.write(chunk) + + if magic.from_file(image_path).lower().find(mimetypes.guess_extension(req.content_type)[1:]) < 0: + raise falcon.HTTPBadRequest( "Memechain error", + "Meme has not passed validation, file extension is wrong.") + + # Check file size meme_filesize = os.path.getsize(image_path) * 0.000001 # in MB @@ -271,18 +311,16 @@ def on_post(self, req, resp): "Meme was not able to be added on IPFS.")) raise falcon.HTTPError(falcon.HTTP_400, "Memechain error", "Meme was not able to be added on IPFS.") - + # Rename local img file to ipfs_id for easy reference new_name = '{img_name}{ext}'.format(img_name=ipfs_id, ext=ext) os.rename(image_path, os.path.join(config['DATA_DIR'], new_name)) - # Add to Kekcoin chain - memetx = MemeTx(ipfs_id) + memetx = MemeTx(ipfs_id, req.params['addr']) prev_block_memes = db.get_prev_block_memes() if prev_block_memes: memetx.generate_hashlink(prev_block_memes) - try: Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes) @@ -293,13 +331,18 @@ def on_post(self, req, resp): logger.error('COMMAND %s Failed %s: %s' % (self.__class__.__name__, 'Memechain Error', - "Meme has not passed memechain validation, file extension not supported.")) + "Meme has not passed memechain validation, file extension not supported.%s") ) raise falcon.HTTPError(falcon.HTTP_400, "Memechain error", - "Meme has not passed validation, file extension not supported.") + "Meme has not passed validation, file extension not supported.%s" % e ) if memetx.is_meme_valid(): + + memetx.blockchain_write() + db.add_meme(**{"ipfs_id": ipfs_id, "hashlink": memetx.get_hashlink(), + "txid": memetx.get_txid(), "author": memetx.get_author(), "block": 0, "imgformat": ext[1:], + "status": "unconfirm"}) resp.status = falcon.HTTP_201 resp.set_header('Powered-By', 'Memechain') @@ -319,7 +362,7 @@ def on_post(self, req, resp): "Meme has not passed validation: ") else: # Genesis block logic - memetx = MemeTx(ipfs_id) + memetx = MemeTx(ipfs_id, req.params['addr']) memetx.generate_genesis_hashlink() memetx.blockchain_write() @@ -358,4 +401,4 @@ def on_post(self, req, resp): app.add_route('/api/getmemeimgbyhash/{ipfs_id}', get_meme_img_by_hash()) # Add meme command -app.add_route('/api/addmeme', add_meme()) \ No newline at end of file +app.add_route('/api/addmeme', add_meme()) diff --git a/config.json b/config.json index c89a43d..5de211a 100644 --- a/config.json +++ b/config.json @@ -1,7 +1,12 @@ { - "DATA_DIR" : "/root/memechain-api/data", + "DATA_DIR" : "/home/parallels/memechain-api/data", "RPC_USER" : "user", "RPC_PASS" : "pass", + "RPC_IP" : "127.0.0.1", "RPC_PORT" : "13377", - "ALLOWED_IP_ADDRESSES" : [] -} \ No newline at end of file + "ALLOWED_IP_ADDRESSES" : [], + "ALLOWED_IMAGE_EXTENSIONS" : ["jpg","png","gif"], + "ENABLE_LOG_MEMTX_NOT_FOUND" : false, + "MULTIPLE_SYNC_RUNNING" : false, + "CHECK_FILES_ON_RUNNING" : true +} diff --git a/lib/blockchain.py b/lib/blockchain.py index a5ff06d..cae5c65 100644 --- a/lib/blockchain.py +++ b/lib/blockchain.py @@ -30,8 +30,7 @@ def get_blockchain_info(): Returns: getinfo output (dict) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.getinfo() @@ -42,8 +41,7 @@ def get_block_height(): Returns: Block height (int) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.getblockcount() @@ -58,8 +56,7 @@ def get_block_txs(height): Returns: List of transaction ids (array) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) block_hash = rpc.getblockhash(height) block = rpc.getblock(block_hash) @@ -67,27 +64,26 @@ def get_block_txs(height): return block['tx'] -def get_input(): +def get_input(addr): """ Method used to get unspent inputs Returns: Transaction object (dict) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) unspent = rpc.listunspent() for tx in unspent: - if float(tx["amount"]) > 0.01: + if float(tx["amount"]) > 0.01 and addr == tx['address']: return tx else: raise Exception( "No valid inputs, inputs must be greater than 0.001 KEK") -def create_raw_op_return_transaction(metadata): +def create_raw_op_return_transaction(metadata,addr): """ Method used to create a transaction with embedded data through OP_RETURN @@ -98,8 +94,7 @@ def create_raw_op_return_transaction(metadata): Raw transaction (hex) Author address (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) if sys.getsizeof(metadata) > MAX_OP_RETURN_BYTES: raise Exception("Metadata size is over MAX_OP_RETURN_BYTES") @@ -108,11 +103,11 @@ def create_raw_op_return_transaction(metadata): raise Exception( "This tool set does not currently support reading op_return data with less than 4 chars") - input_tx = get_input() + input_tx = get_input(addr) init_raw_tx = rpc.createrawtransaction([{"txid": input_tx["txid"], "vout": input_tx["vout"]}], { rpc.getnewaddress(): TX_BURN_AMOUNT, input_tx["address"]: round(float(input_tx["amount"]) - TX_BURN_AMOUNT - TX_FEE_RATE, 8)}) - + for vout in rpc.decoderawtransaction(init_raw_tx)["vout"]: if float(vout["value"]) == TX_BURN_AMOUNT: oldScriptPubKey = "19" + vout["scriptPubKey"]["hex"] @@ -138,8 +133,7 @@ def sign_raw_transaction(tx): Returns: Signed raw transaction (hex) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.signrawtransaction(tx)["hex"] @@ -154,8 +148,7 @@ def send_raw_transaction(tx_hex): Returns: Transaction id (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) return rpc.sendrawtransaction(tx_hex) @@ -171,8 +164,7 @@ def get_op_return_data(txid): Embedded metadata (str) Author address (str) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) raw_tx = rpc.getrawtransaction(txid) tx_data = rpc.decoderawtransaction(raw_tx) @@ -191,7 +183,7 @@ def get_op_return_data(txid): author = None else: op_return_data = None - author = None + author = None return op_return_data, author @@ -205,8 +197,7 @@ def get_tx_burn_amount(txid): Returns: Sum of input values, i.e. burn amount (float) """ - rpc = AuthServiceProxy(("http://%s:%s@127.0.0.1:%s/") % - (config['RPC_USER'], config['RPC_PASS'], config['RPC_PORT'])) + rpc = AuthServiceProxy(("http://%s:%s@%s:%s/") % (config['RPC_USER'], config['RPC_PASS'], config['RPC_IP'], config['RPC_PORT'])) raw_tx = rpc.getrawtransaction(txid) tx_data = rpc.decoderawtransaction(raw_tx) diff --git a/lib/db.py b/lib/db.py index 53c9c48..df6c765 100644 --- a/lib/db.py +++ b/lib/db.py @@ -43,9 +43,9 @@ class MemeChainDB(object): def __init__(self, db_path): self._db = TinyDB(db_path) - def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author): + def add_meme(self, ipfs_id, hashlink, txid, block, imgformat, author, status): self._db.insert({"ipfs_id": ipfs_id, "hashlink": hashlink, - "txid": txid, "block": block, "imgformat" : imgformat, "author" : author}) + "txid": txid, "block": block, "imgformat" : imgformat, "author" : author, "status": status}) def remove_meme(self, ipfs_id): self._db.remove(Query().ipfs_id == ipfs_id) @@ -53,6 +53,14 @@ def remove_meme(self, ipfs_id): def get_memechain_height(self): return len(self._db) + def update_meme(self, ipfs_id, block): + memes = self._db.search(Query().ipfs_id == ipfs_id) + print(memes) + for meme in memes: + meme["block"] = block + meme["status"] = "confirm" + self._db.write_back(memes) + def search_by_block(self, block): """ Get a meme entry using block number as the search parameter @@ -119,5 +127,8 @@ def get_prev_block_memes(self): def get_last_meme(self): return Index(self._db)[-1] + def get_all_memes(self): + return self._db.search(Query().ipfs_id.exists()) + def get_meme_height_by_ipfs_id(self, ipfs_id): return Index(self._db).return_index(self._db.get(Query().ipfs_id == ipfs_id)) + 1 diff --git a/lib/memechain.py b/lib/memechain.py index 2997434..6bcd1dd 100644 --- a/lib/memechain.py +++ b/lib/memechain.py @@ -1,6 +1,6 @@ import os from hashlib import sha256 - +from logger import * from .blockchain import * from .ipfs import IPFSTools @@ -14,13 +14,13 @@ def __init__(self, MemeTX, db, ipfs_dir, prev_block_memes, sync = False, genesis if genesis == False: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir), self.is_valid_hash_link(MemeTX, prev_block_memes), - self.check_duplicate(db, MemeTX.get_ipfs_id())] + self.check_duplicate(db, MemeTX.get_ipfs_id()), + self.check_file_ext(MemeTX.get_ipfs_id())] else: self.is_valid = [self.check_ipfs_existance(MemeTX.get_ipfs_id(), ipfs_dir)] if sync == True: self.is_valid.append(self.check_burn_amount(MemeTX)) - MemeTX.set_is_valid(self.is_valid) def is_valid_hash_link(self, MemeTX, prev_block_memes): @@ -33,6 +33,7 @@ def is_valid_hash_link(self, MemeTX, prev_block_memes): raw_str += ''.join(meme['ipfs_id'] for meme in prev_block_memes) hashlink = sha256(raw_str.encode('utf-8')).hexdigest()[:16] if hashlink != MemeTX.get_hashlink(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Hashlink %s." % hashlink)) return False else: return True @@ -52,6 +53,7 @@ def check_ipfs_existance(self, ipfs_id, ipfs_dir): # IPFS Tools should be instanciated. ipfs = IPFSTools() if not ipfs.get_meme(ipfs_id, ipfs_dir): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme does not exist on IPFS yet %s." % ipfs_id)) return False # Meme does not exist on IPFS yet else: return True # Meme already exists on global IPFS @@ -65,8 +67,8 @@ def check_duplicate(self, db, ipfs_id): ipfs_id - attribute of MemeTX """ meme = db.search_by_ipfs_id(ipfs_id) - - if meme: + if meme and meme["status"] == "confirm": + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Meme exist on db yet %s." % ipfs_id)) return False else: return True @@ -76,33 +78,44 @@ def check_burn_amount(self, MemeTX): Checks whether the correct amount of KEKs were burned for the MemeTx. """ burn_amount = get_tx_burn_amount(MemeTX.get_txid()) - if float(burn_amount) == TX_BURN_AMOUNT: return True else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct amount of KEKs were burned for the MemeTx %s." % MemeTX.get_ipfs_id())) return False + def check_file_ext(self, meme): + meme_filepath = IPFSTools().get_meme(meme, config['DATA_DIR']) + ext = meme_filepath.split(".")[-1] + if ext in config["ALLOWED_IMAGE_EXTENSIONS"]: + return True + else: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Not correct image file extension %s." % ext)) + return False class MemeTx(object): """ MemeChain TX object. Used to construct a MemeChainTX. """ - def __init__(self, ipfs_id): + def __init__(self, ipfs_id, addr=''): # Identifier is first 4 letters of the SHA256 hash of KEK self._identifier = '3ae4' self.command_bytes = '00' self.ipfs_id = ipfs_id self._is_valid = False + self.addr = addr def set_is_valid(self, values): self._is_valid = values def is_meme_valid(self): - if False not in self._is_valid: - return True - else: - return False + + for num, value in enumerate(self._is_valid, start=1): + if not value: + return num + + return -1 def get_ipfs_id(self): return self.ipfs_id @@ -140,7 +153,6 @@ def generate_hashlink(self, prev_block_memes): def blockchain_write(self): metadata = self._identifier + self.command_bytes + self.ipfs_id + self.hashlink - - rawtx, self.author = create_raw_op_return_transaction(metadata) + rawtx, self.author = create_raw_op_return_transaction(metadata, self.addr) signedtx = sign_raw_transaction(rawtx) self.txid = send_raw_transaction(signedtx) diff --git a/sync.py b/sync.py index ed0cb69..33369ca 100644 --- a/sync.py +++ b/sync.py @@ -2,13 +2,16 @@ import os import json import pickle as pickle - +import time from lib.db import MemeChainDB from lib.blockchain import * from lib.memechain import MemeTx, Validate from lib.ipfs import IPFSTools from lib.exceptions import * +import psutil + + # Load configuration file with open("config.json", "r") as f: config = json.loads(f.read()) @@ -79,7 +82,6 @@ def return_memetxs(self): def sync_block(db, block): parser = MemechainParser(block) parser.collect_memetxs() - memetxs = parser.return_memetxs() if memetxs: @@ -89,36 +91,72 @@ def sync_block(db, block): memetx = MemeTx(meme['ipfs_id']) memetx.generate_hashlink(prev_block_memes) memetx.txid = meme['txid'] - Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=prev_block_memes, sync=True) - - if memetx.is_meme_valid(): + valid_state = memetx.is_meme_valid() + if valid_state == -1: meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) ext = meme_filepath.split(".")[-1] - if ext in ["jpg", "png", "gif"]: + if db.search_by_ipfs_id(meme['ipfs_id']): + db.update_meme(meme['ipfs_id'], block) + logger.info('COMMAND %s Success %s: %s' % ( + 'Sync', 'Memechain', "Meme %s update in database." % meme['ipfs_id'])) + else: db.add_meme(**{"ipfs_id": meme['ipfs_id'], "hashlink": meme['hashlink'], - "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext}) - + "txid": meme['txid'], "author": meme['author'], "block": block, "imgformat": ext, "status": "confirm"}) logger.info('COMMAND %s Success %s: %s' % ('Sync', 'Memechain', "Meme %s added to database." % meme['ipfs_id'])) - - else: - # Delete invalid Meme - os.remove(meme_filepath) - - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid Meme image extension %s." % ext)) - - else: + elif valid_state != 3: + meme_filepath = IPFSTools().get_meme(meme['ipfs_id'], config['DATA_DIR']) + os.remove(meme_filepath) logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid MemeTx %s." % meme['ipfs_id'])) else: - logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + if config["ENABLE_LOG_MEMTX_NOT_FOUND"]: + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "No Meme TXs found in block %s." % block)) + +def check_files_status(db): + for file in os.listdir(config['DATA_DIR']): + if file.endswith(tuple(config["ALLOWED_IMAGE_EXTENSIONS"])): + if db.search_by_ipfs_id(file.split(".")[0]) is None: + os.remove(os.path.join(config['DATA_DIR'], file)) + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Remove file from data folder (not exist in db) %s." % file)) + + for meme in db.get_all_memes(): + if not os.path.isfile(os.path.join(config['DATA_DIR'], meme["ipfs_id"] + "." + meme["imgformat"])): + logger.info('COMMAND %s Failed %s: %s' % ( + 'Sync', 'Memechain', "File not found in data folder (exist in db) %s." % meme["ipfs_id"])) + ipfs = IPFSTools() + ipfs.get_meme(meme["ipfs_id"], config['DATA_DIR']) + logger.info('COMMAND %s Info %s: %s' % ('Sync', 'Memechain', "Try downloading missing file with id %s." % meme["ipfs_id"])) + + +def check_running(): + counter = 0 + for q in psutil.process_iter(): + if q.name().find('sync'): + if len(q.cmdline())>1 and 'sync.py' in q.cmdline()[1]: + counter = counter + 1 + if counter > 1: + return True + else: + return False + + if __name__ == '__main__': # Load database db = MemeChainDB(os.path.join(config['DATA_DIR'], 'memechain.json')) - + + + + if not config["MULTIPLE_SYNC_RUNNING"] and check_running(): + logger.info('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Sync process already running.Shutdown current sync")) + sys.exit(1) + + if config["CHECK_FILES_ON_RUNNING"]: + check_files_status(db) + # Check blockheight block_height = get_block_height() @@ -126,23 +164,25 @@ def sync_block(db, block): # Load genesis meme genesis_meme = GenesisMeme() - memetx = MemeTx(genesis_meme.genesis_ipfs_id) + memetx = MemeTx(genesis_meme.genesis_ipfs_id,genesis_meme.genesis_author) memetx.generate_genesis_hashlink() memetx.txid = genesis_meme.genesis_txid - + Validate(memetx, db=db, ipfs_dir=config['DATA_DIR'], prev_block_memes=[], sync=True, genesis=True) - # Add genesis meme to database + + db.add_meme(**{"ipfs_id": genesis_meme.get_ipfs_id(), "hashlink": genesis_meme.get_hashlink(), - "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format}) + "txid": genesis_meme.genesis_txid, "author": genesis_meme.genesis_author, "block": genesis_meme.genesis_kekcoin_block, "imgformat": genesis_meme.genesis_img_format, "status": "confirm"}) # Sync loop if genesis_meme.genesis_kekcoin_block < block_height: - for block in range(genesis_meme.genesis_kekcoin_block + 1, block_height + 1): + block = genesis_meme.genesis_kekcoin_block + 1 + max_errors = 0 + while block < block_height + 1: try: sync_block(db, block) - except InvalidMultihashError as e: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) @@ -159,9 +199,21 @@ def sync_block(db, block): except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block - 1, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) - + + except IOError as e: + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 + except KeyboardInterrupt: + # Dump current sync height into a pickle + pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + block = block + 1 # Dump current sync height into a pickle - pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) else: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Blockchain Error', "Kekcoin blockchain syncing...")) @@ -175,11 +227,15 @@ def sync_block(db, block): synced_height = last_meme['block'] # Sync loop + if synced_height < block_height: - for block in range(synced_height + 1, block_height + 1): + block = synced_height + 1 + max_errors = 0 + while block < block_height + 1: try: + pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) sync_block(db, block) - + except InvalidMultihashError as e: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.")) @@ -196,9 +252,20 @@ def sync_block(db, block): except KeyboardInterrupt: # Dump current sync height into a pickle pickle.dump(block - 1, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + + except IOError as e: + logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Memechain', "Invalid ipfs multihash.%s") % e) + if max_errors < 10: + time.sleep(10) + block = block - 1 + max_errors = max_errors + 1 + else: + max_errors = 0 + + block = block + 1 # Dump current sync height into a pickle - pickle.dump(block_height, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) + pickle.dump(block, open(os.path.join(config['DATA_DIR'], 'sync.p'), 'wb')) else: logger.error('COMMAND %s Failed %s: %s' % ('Sync', 'Blockchain Error', "Kekcoin blockchain syncing..."))