[alpha] move from twisted to asyncio + new network

.travis.yml

@@ -1,6 +1,10 @@
 language: python
-python:
-  - "3.6"
+
+matrix:
+  include:
+    - python: 3.7
+      dist: xenial
+      sudo: true
 
 sudo: required
 services:

CHANGELOG.rst

@@ -16,6 +16,7 @@ All notable changes to this project are documented in this file.
 - Fixed size calculation for `InvocationTransaction` `#919 <https://github.com/CityOfZion/neo-python/pull/919>`_
 - Update Virtual Machine to latest implementation, Add support for running official JSON test vectors `#921 <https://github.com/CityOfZion/neo-python/pull/921>`_
 - Add PICKITEM for ByteArray into VM `#923 <https://github.com/CityOfZion/neo-python/pull/923>`_
+- Fix sys_fee calculation in block persist.
 
 
 [0.8.4] 2019-02-14

docs/source/install.rst

@@ -139,6 +139,24 @@ The solution probably is
 
     brew reinstall openssl
 
+-----
+
+If you encounter an issue installing the ``scrypt`` module (possibly after updating OSX) with an error like this:
+
+.. code-block:: sh
+
+    ld: library not found for -lcrypto
+    clang: error: linker command failed with exit code 1 (use -v to see invocation)
+    error: command 'gcc' failed with exit status 1
+
+The solution probably is
+
+.. code-block:: sh
+
+    $ brew install openssl
+    $ export CFLAGS="-I$(brew --prefix openssl)/include $CFLAGS"
+    $ export LDFLAGS="-L$(brew --prefix openssl)/lib $LDFLAGS"
+
 
 Install from PyPi
 =================

examples/node.py

@@ -1,18 +1,16 @@
 """
-Minimal NEO node with custom code in a background thread.
+Minimal NEO node with custom code in a background task.
 
 It will log events from all smart contracts on the blockchain
 as they are seen in the received blocks.
 """
-import threading
-from time import sleep
+import asyncio
 
 from logzero import logger
-from twisted.internet import reactor, task
 
-from neo.Network.NodeLeader import NodeLeader
 from neo.Core.Blockchain import Blockchain
 from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain
+from neo.Network.p2pservice import NetworkService
 from neo.Settings import settings
 
 
@@ -21,16 +19,11 @@
 # settings.set_logfile("/tmp/logfile.log", max_bytes=1e7, backup_count=3)
 
 
-def custom_background_code():
-    """ Custom code run in a background thread.
-
-    This function is run in a daemonized thread, which means it can be instantly killed at any
-    moment, whenever the main thread quits. If you need more safety, don't use a  daemonized
-    thread and handle exiting this thread in another way (eg. with signals and events).
-    """
+async def custom_background_code():
+    """ Custom code run in the background."""
     while True:
         logger.info("Block %s / %s", str(Blockchain.Default().Height), str(Blockchain.Default().HeaderHeight))
-        sleep(15)
+        await asyncio.sleep(15)
 
 
 def main():
@@ -40,18 +33,17 @@ def main():
     # Setup the blockchain
     blockchain = LevelDBBlockchain(settings.chain_leveldb_path)
     Blockchain.RegisterBlockchain(blockchain)
-    dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
-    dbloop.start(.1)
-    NodeLeader.Instance().Start()
-
-    # Start a thread with custom code
-    d = threading.Thread(target=custom_background_code)
-    d.setDaemon(True)  # daemonizing the thread will kill it when the main thread is quit
-    d.start()
-
-    # Run all the things (blocking call)
-    reactor.run()
-    logger.info("Shutting down.")
+
+    loop = asyncio.get_event_loop()
+    # Start a reoccurring task with custom code
+    loop.create_task(custom_background_code())
+    p2p = NetworkService()
+    loop.create_task(p2p.start())
+
+    # block from here on
+    loop.run_forever()
+
+    # have a look at the other examples for handling graceful shutdown.
 
 
 if __name__ == "__main__":

examples/smart-contract-rest-api.py

@@ -6,8 +6,7 @@
 Execution.Success and several more. See the documentation here:
 http://neo-python.readthedocs.io/en/latest/smartcontracts.html
 
-This example requires the environment variable NEO_REST_API_TOKEN, and can
-optionally use NEO_REST_LOGFILE and NEO_REST_API_PORT.
+This example optionally uses the environment variables NEO_REST_LOGFILE and NEO_REST_API_PORT.
 
 Example usage (with "123" as valid API token):
 
@@ -18,29 +17,21 @@
     $ curl localhost:8080
     $ curl -H "Authorization: Bearer 123" localhost:8080/echo/hello123
     $ curl -X POST -H "Authorization: Bearer 123" -d '{ "hello": "world" }' localhost:8080/echo-post
-
-The REST API is using the Python package 'klein', which makes it possible to
-create HTTP routes and handlers with Twisted in a similar style to Flask:
-https://github.com/twisted/klein
 """
+import asyncio
 import os
-import threading
-import json
-from time import sleep
+from contextlib import suppress
+from signal import SIGINT
 
+from aiohttp import web
 from logzero import logger
-from twisted.internet import reactor, task, endpoints
-from twisted.web.server import Request, Site
-from klein import Klein, resource
 
-from neo.Network.NodeLeader import NodeLeader
 from neo.Core.Blockchain import Blockchain
 from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain
+from neo.Network.p2pservice import NetworkService
 from neo.Settings import settings
-
-from neo.Network.api.decorators import json_response, gen_authenticated_decorator, catch_exceptions
-from neo.contrib.smartcontract import SmartContract
 from neo.SmartContract.ContractParameter import ContractParameter, ContractParameterType
+from neo.contrib.smartcontract import SmartContract
 
 # Set the hash of your contract here:
 SMART_CONTRACT_HASH = "6537b4bd100e514119e3a7ab49d520d20ef2c2a4"
@@ -56,19 +47,9 @@
 if LOGFILE:
     settings.set_logfile(LOGFILE, max_bytes=1e7, backup_count=3)
 
-# Internal: get the API token from an environment variable
-API_AUTH_TOKEN = os.getenv("NEO_REST_API_TOKEN", None)
-if not API_AUTH_TOKEN:
-    raise Exception("No NEO_REST_API_TOKEN environment variable found!")
-
 # Internal: setup the smart contract instance
 smart_contract = SmartContract(SMART_CONTRACT_HASH)
 
-# Internal: setup the klein instance
-app = Klein()
-
-# Internal: generate the @authenticated decorator with valid tokens
-authenticated = gen_authenticated_decorator(API_AUTH_TOKEN)
 
 #
 # Smart contract event handler for Runtime.Notify events
@@ -92,7 +73,7 @@ def sc_notify(event):
 #
 # Custom code that runs in the background
 #
-def custom_background_code():
+async def custom_background_code():
     """ Custom code run in a background thread. Prints the current block height.
 
     This function is run in a daemonized thread, which means it can be instantly killed at any
@@ -101,78 +82,108 @@ def custom_background_code():
     """
     while True:
         logger.info("Block %s / %s", str(Blockchain.Default().Height), str(Blockchain.Default().HeaderHeight))
-        sleep(15)
+        await asyncio.sleep(15)
 
 
 #
 # REST API Routes
 #
-@app.route('/')
-def home(request):
-    return "Hello world"
-
-
-@app.route('/echo/<msg>')
-@catch_exceptions
-@authenticated
-@json_response
-def echo_msg(request, msg):
-    return {
-        "echo": msg
+async def home_route(request):
+    return web.Response(body="hello world")
+
+
+async def echo_msg(request):
+    res = {
+        "echo": request.match_info['msg']
     }
+    return web.json_response(data=res)
 
 
-@app.route('/echo-post', methods=['POST'])
-@catch_exceptions
-@authenticated
-@json_response
-def echo_post(request):
+async def echo_post(request):
     # Parse POST JSON body
-    body = json.loads(request.content.read().decode("utf-8"))
+
+    body = await request.json()
 
     # Echo it
-    return {
+    res = {
         "post-body": body
     }
+    return web.json_response(data=res)
+
 
 #
-# Main method which starts everything up
+# Main setup method
 #
 
-
-def main():
+async def setup_and_start(loop):
     # Use TestNet
-    settings.setup_testnet()
+    settings.setup_privnet()
 
     # Setup the blockchain
     blockchain = LevelDBBlockchain(settings.chain_leveldb_path)
     Blockchain.RegisterBlockchain(blockchain)
-    dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
-    dbloop.start(.1)
-    NodeLeader.Instance().Start()
+
+    p2p = NetworkService()
+    loop.create_task(p2p.start())
+    bg_task = loop.create_task(custom_background_code())
 
     # Disable smart contract events for external smart contracts
     settings.set_log_smart_contract_events(False)
 
-    # Start a thread with custom code
-    d = threading.Thread(target=custom_background_code)
-    d.setDaemon(True)  # daemonizing the thread will kill it when the main thread is quit
-    d.start()
-
-    # Hook up Klein API to Twisted reactor.
-    endpoint_description = "tcp:port=%s:interface=localhost" % API_PORT
+    app = web.Application()
+    app.add_routes([
+        web.route('*', '/', home_route),
+        web.get("/echo-get/{msg}", echo_msg),
+        web.post("/echo-post/", echo_post),
+    ])
 
-    # If you want to make this service externally available (not only at localhost),
-    # then remove the `interface=localhost` part:
-    # endpoint_description = "tcp:port=%s" % API_PORT
-
-    endpoint = endpoints.serverFromString(reactor, endpoint_description)
-    endpoint.listen(Site(app.resource()))
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, "0.0.0.0", API_PORT)
+    await site.start()
 
     # Run all the things (blocking call)
     logger.info("Everything setup and running. Waiting for events...")
-    reactor.run()
-    logger.info("Shutting down.")
+    return site
+
+
+async def shutdown():
+    # cleanup any remaining tasks
+    for task in asyncio.Task.all_tasks():
+        with suppress((asyncio.CancelledError, Exception)):
+            task.cancel()
+            await task
+
+
+def system_exit():
+    raise SystemExit
+
+
+def main():
+    loop = asyncio.get_event_loop()
+
+    # because a KeyboardInterrupt is so violent it can shutdown the DB in an unpredictable state.
+    loop.add_signal_handler(SIGINT, system_exit)
+
+    main_task = loop.create_task(setup_and_start(loop))
+
+    try:
+        loop.run_forever()
+    except SystemExit:
+        logger.info("Shutting down...")
+        site = main_task.result()
+        loop.run_until_complete(site.stop())
+
+        p2p = NetworkService()
+        loop.run_until_complete(p2p.shutdown())
+
+        loop.run_until_complete(shutdown())
+        loop.stop()
+    finally:
+        loop.close()
+
+    logger.info("Closing databases...")
+    Blockchain.Default().Dispose()
 
 
 if __name__ == "__main__":