diff --git a/.travis.yml b/.travis.yml index f8eaa97..f31e77a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,9 @@ sudo: required language: python +dist: trusty python: - "2.7" - - "3.3" - - "3.4" - - "3.5" env: global: @@ -15,64 +13,63 @@ env: - INCLUDE_DIR=./libevmjit_ext/include:/opt/gcc-4.6.3/include/c++/4.6.3:/opt/gcc-4.6.3/include/c++/4.6.3/x86_64-unknown-linux-gnu - WHEELBUILDER_IMAGE=quay.io/pypa/manylinux1_x86_64 - PYPI_USERNAME=insert_pypi_upload_username_here - matrix: - - BUNDLED=0 - - BUNDLED=1 + # matrix: + # - BUNDLED=0 + # - BUNDLED=1 cache: + ccache: true directories: - $HOME/.cache/pip - $HOME/.cache/python-dl + - build/evmjit/build # Sigh... Until Travis issue #2312 is fixed Travis' osx image doesn't properly # support Python so we need to manually specify this matrix... -matrix: - include: - - os: osx - language: generic - osx_image: xcode7.1 - python: 2.7 - env: - - TRAVIS_PYTHON_VERSION=2.7 - - BUNDLED=1 - - os: osx - language: generic - osx_image: xcode7.1 - python: 3.4 - env: - - TRAVIS_PYTHON_VERSION=3.4 - - BUNDLED=1 - - os: osx - language: generic - osx_image: xcode7.1 - python: 3.5 - env: - - TRAVIS_PYTHON_VERSION=3.5 - - BUNDLED=1 - # Build `manylinux1` wheels - - os: linux - language: python - python: 3.5 - sudo: required - services: - - docker - env: - - BUNDLED=1 - - BUILD_LINUX_WHEELS=1 +# matrix: +# include: +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 2.7 +# env: +# - TRAVIS_PYTHON_VERSION=2.7 +# - BUNDLED=1 +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 3.4 +# env: +# - TRAVIS_PYTHON_VERSION=3.4 +# - BUNDLED=1 +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 3.5 +# env: +# - TRAVIS_PYTHON_VERSION=3.5 +# - BUNDLED=1 +# # Build `manylinux1` wheels +# - os: linux +# language: python +# python: 3.5 +# sudo: required +# services: +# - docker +# env: +# - BUNDLED=1 +# - BUILD_LINUX_WHEELS=1 addons: apt: packages: - git - - libtool - cmake - - pkg-config - - libffi-dev - - libgmp-dev - - libatomic-ops-dev before_install: - - source .travis/install.sh + - git clone --depth 1 https://github.com/ethereum/tests ~/ethereum-tests + - export ETHEREUM_TEST_PATH=~/ethereum-tests + - pip install pytest sha3 install: - python setup.py install @@ -86,13 +83,14 @@ install: # into producing correct paths. For that to work we need to re-rename the # source back to it's correct name. script: - - mv evmjit _evmjit - - coverage run --parallel --include="*/site-packages/*.egg/evmjit/*" -m py.test - - mv _evmjit evmjit - - coverage combine + # - mv evmjit _evmjit + # - coverage run --parallel --include="*/site-packages/*.egg/evmjit/*" -m py.test + # - mv _evmjit evmjit + # - coverage combine + - py.test after_success: - - coveralls + # - coveralls deploy: provider: script diff --git a/_cffi_build/build.py b/_cffi_build/build.py deleted file mode 100755 index aa427bc..0000000 --- a/_cffi_build/build.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import sys -from collections import namedtuple - -from cffi import FFI - -sys.path.append(os.path.abspath(os.path.dirname(__file__))) - - -def absolute(*paths): - op = os.path - return op.realpath(op.abspath(op.join(op.dirname(__file__), *paths))) - -Source = namedtuple('Source', ('h', 'include')) - - -def _mk_ffi(sources, name="_libevmjit", bundled=True, **kwargs): - ffi = FFI() - code = [] - if 'INCLUDE_DIR' in os.environ: - kwargs['include_dirs'] = [absolute(os.environ['INCLUDE_DIR'])] - if 'LIB_DIR' in os.environ: - kwargs['library_dirs'] = [absolute(os.environ['LIB_DIR'])] - for source in sources: - with open(source.h, 'rt') as h: - ffi.cdef(h.read()) - code.append(source.include) - if bundled: - code.append("#define PY_USE_BUNDLED") - ffi.set_source(name, "\n".join(code), **kwargs) - return ffi - - -_base = [Source(absolute("evm.h"), "#include \"_cffi_build/evm.h\"",)] -ffi = _mk_ffi(_base, libraries=['evmjit-standalone', 'stdc++']) -ffi.cdef(""" - extern "Python" union evm_variant evm_query(struct evm_env* env, - enum evm_query_key key, - union evm_variant arg); - extern "Python" void evm_update(struct evm_env* env, - enum evm_update_key key, - union evm_variant arg1, - union evm_variant arg2); - extern "Python" int64_t evm_call( - struct evm_env* env, - enum evm_call_kind kind, - int64_t gas, - struct evm_hash160 address, - struct evm_uint256 value, - uint8_t const* input, - size_t input_size, - uint8_t* output, - size_t output_size); -""") -ffi.compile() diff --git a/_cffi_build/evm.h b/_cffi_build/evm.h deleted file mode 100755 index 6fe302d..0000000 --- a/_cffi_build/evm.h +++ /dev/null @@ -1,436 +0,0 @@ -/// EVM-C -- C interface to Ethereum Virtual Machine -/// -/// ## High level design rules -/// 1. Pass function arguments and results by value. -/// This rule comes from modern C++ and tries to avoid costly alias analysis -/// needed for optimization. As the result we have a lots of complex structs -/// and unions. And variable sized arrays of bytes cannot be passed by copy. -/// 2. The EVM operates on integers so it prefers values to be host-endian. -/// On the other hand, LLVM can generate good code for byte swaping. -/// The interface also tries to match host application "natural" endianess. -/// I would like to know what endianess you use and where. -/// -/// @defgroup EVMC EVM-C -/// @{ -// #ifndef EVM_H -// #define EVM_H -// -// #include // Definition of int64_t, uint64_t. -// #include // Definition of size_t. -// -// #if __cplusplus -// extern "C" { -// #endif - -/// The EVM-C ABI version number matching the interface declared in this file. -static const uint32_t EVM_ABI_VERSION = 0; - -/// Host-endian 256-bit integer. -/// -/// 32 bytes of data representing host-endian (that means little-endian almost -/// all the time) 256-bit integer. This applies to the words[] order as well. -/// words[0] contains the 64 lowest precision bits, words[3] constains the 64 -/// highest precision bits. -struct evm_uint256 { - /// The 4 64-bit words of the integer. Memory aligned to 8 bytes. - uint64_t words[4]; -}; - -/// Big-endian 160-bit hash suitable for keeping an Ethereum address. -struct evm_hash160 { - /// The 20 bytes of the hash. - uint8_t bytes[20]; -}; - - -/// Big-endian 256-bit integer/hash. -/// -/// 32 bytes of data. For EVM that means big-endian 256-bit integer. Values of -/// this type are converted to host-endian values inside EVM. -struct evm_hash256 { - /// The 32 bytes of the integer/hash. - /// - /// The memory is expected be aligned to 8 bytes, but there is no portable - /// way to express that. - uint8_t bytes[32]; -}; - -/// The execution result code. -enum evm_result_code { - EVM_SUCCESS = 0, ///< Execution finished with success. - EVM_FAILURE = 1, ///< Generic execution failure. - EVM_OUT_OF_GAS = 2, - EVM_BAD_INSTRUCTION = 3, - EVM_BAD_JUMP_DESTINATION = 4, - EVM_STACK_OVERFLOW = 5, - EVM_STACK_UNDERFLOW = 6, -}; - -/// The EVM code execution result. -struct evm_result { - /// The execution result code. - enum evm_result_code code; - - /// The amount of gas left after the execution. - /// - /// The value is valid only if evm_result::code == ::EVM_SUCCESS. - int64_t gas_left; - - /// The reference to output data. The memory containing the output data - /// is owned by EVM and is freed with evm_release_result_fn(). - uint8_t const* output_data; - - /// The size of the output data. - size_t output_size; - - /// @name Optional - /// The optional information that EVM is not required to provide. - /// @{ - - /// The pointer to EVM-owned memory. For EVM internal use. - /// @see output_data. - void* internal_memory; - - /// The error message explaining the result code. - char const* error_message; - - /// @} -}; - -/// The query callback key. -enum evm_query_key { - EVM_SLOAD = 0, ///< Storage value of a given key for SLOAD. - EVM_ADDRESS = 1, ///< Address of the contract for ADDRESS. - EVM_CALLER = 2, ///< Message sender address for CALLER. - EVM_ORIGIN = 3, ///< Transaction origin address for ORIGIN. - EVM_GAS_PRICE = 4, ///< Transaction gas price for GASPRICE. - EVM_COINBASE = 5, ///< Current block miner address for COINBASE. - EVM_DIFFICULTY = 6, ///< Current block difficulty for DIFFICULTY. - EVM_GAS_LIMIT = 7, ///< Current block gas limit for GASLIMIT. - EVM_NUMBER = 8, ///< Current block number for NUMBER. - EVM_TIMESTAMP = 9, ///< Current block timestamp for TIMESTAMP. - EVM_CODE_BY_ADDRESS = 10, ///< Code by an address for EXTCODE/SIZE. - EVM_BALANCE = 11, ///< Balance of a given address for BALANCE. - EVM_BLOCKHASH = 12 ///< Block hash of by block number for BLOCKHASH. -}; - - -/// Opaque struct representing execution enviroment managed by the host -/// application. -struct evm_env; - -/// Variant type to represent possible types of values used in EVM. -/// -/// Type-safety is lost around the code that uses this type. We should have -/// complete set of unit tests covering all possible cases. -/// The size of the type is 64 bytes and should fit in single cache line. -union evm_variant { - /// A host-endian 64-bit integer. - int64_t int64; - - /// A host-endian 256-bit integer. - struct evm_uint256 uint256; - - /// A big-endian 256-bit integer/hash. - struct evm_hash256 hash256; - - struct { - /// Additional padding to align the evm_variant::address with lower - /// bytes of a full 256-bit hash. - uint8_t address_padding[12]; - - /// An Ethereum address. - struct evm_hash160 address; - }; - - /// A memory reference. - struct { - /// Pointer to the data. - uint8_t const* data; - - /// Size of the referenced memory/data. - size_t data_size; - }; -}; - -/// Query callback function. -/// -/// This callback function is used by the EVM to query the host application -/// about additional data required to execute EVM code. -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param key The kind of the query. See evm_query_key and details below. -/// @param arg Additional argument to the query. It has defined value only for -/// the subset of query keys. -/// -/// ## Types of queries -/// Key | Arg | Expected result -/// ----------------------| -------------------- | ---------------------------- -/// ::EVM_GAS_PRICE | | evm_variant::uint256 -/// ::EVM_ADDRESS | | evm_variant::address -/// ::EVM_CALLER | | evm_variant::address -/// ::EVM_ORIGIN | | evm_variant::address -/// ::EVM_COINBASE | | evm_variant::address -/// ::EVM_DIFFICULTY | | evm_variant::uint256 -/// ::EVM_GAS_LIMIT | | evm_variant::uint256 -/// ::EVM_NUMBER | | evm_variant::int64? -/// ::EVM_TIMESTAMP | | evm_variant::int64? -/// ::EVM_CODE_BY_ADDRESS | evm_variant::address | evm_variant::data -/// ::EVM_BALANCE | evm_variant::address | evm_variant::uint256 -/// ::EVM_BLOCKHASH | evm_variant::int64 | evm_variant::hash256 -/// ::EVM_SLOAD | evm_variant::uint256 | evm_variant::uint256? -typedef union evm_variant (*evm_query_fn)(struct evm_env* env, - enum evm_query_key key, - union evm_variant arg); - -/// The update callback key. -enum evm_update_key { - EVM_SSTORE = 0, ///< Update storage entry - EVM_LOG = 1, ///< Log. - EVM_SELFDESTRUCT = 2, ///< Mark contract as selfdestructed and set - /// beneficiary address. -}; - - -/// Update callback function. -/// -/// This callback function is used by the EVM to modify contract state in the -/// host application. -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param key The kind of the update. See evm_update_key and details below. -/// -/// ## Kinds of updates -/// -/// - ::EVM_SSTORE -/// @param arg1 evm_variant::uint256 The index of the storage entry. -/// @param arg2 evm_variant::uint256 The value to be stored. -/// -/// - ::EVM_LOG -/// @param arg1 evm_variant::data The log unindexed data. -/// @param arg2 evm_variant::data The log topics. The referenced data is an -/// array of evm_hash256[] of possible length -/// from 0 to 4. So the valid -/// evm_variant::data_size values are 0, 32, 64 -/// 92 and 128. -/// -/// - ::EVM_SELFDESTRUCT -/// @param arg1 evm_variant::address The beneficiary address. -typedef void (*evm_update_fn)(struct evm_env* env, - enum evm_update_key key, - union evm_variant arg1, - union evm_variant arg2); - -/// The kind of call-like instruction. -enum evm_call_kind { - EVM_CALL = 0, ///< Request CALL. - EVM_DELEGATECALL = 1, ///< Request DELEGATECALL. The value param ignored. - EVM_CALLCODE = 2, ///< Request CALLCODE. - EVM_CREATE = 3 ///< Request CREATE. Semantic of some params changes. -}; - -/// The flag indicating call failure in evm_call_fn(). -static const int64_t EVM_CALL_FAILURE = INT64_MIN; - -/// Pointer to the callback function supporting EVM calls. -/// -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param kind The kind of call-like opcode requested. -/// @param gas The amount of gas for the call. -/// @param address The address of a contract to be called. Ignored in case -/// of CREATE. -/// @param value The value sent to the callee. The endowment in case of -/// CREATE. -/// @param input The call input data or the CREATE init code. -/// @param input_size The size of the input data. -/// @param output The reference to the memory where the call output is to -/// be copied. In case of CREATE, the memory is guaranteed -/// to be at least 20 bytes to hold the address of the -/// created contract. -/// @param output_data The size of the output data. In case of CREATE, expected -/// value is 20. -/// @return If non-negative - the amount of gas left, -/// If negative - an exception occurred during the call/create. -/// There is no need to set 0 address in the output in this case. -typedef int64_t (*evm_call_fn)( - struct evm_env* env, - enum evm_call_kind kind, - int64_t gas, - struct evm_hash160 address, - struct evm_uint256 value, - uint8_t const* input, - size_t input_size, - uint8_t* output, - size_t output_size); - - -/// Opaque type representing a EVM instance. -struct evm_instance; - -/// Creates new EVM instance. -/// -/// Creates new EVM instance. The instance must be destroyed in evm_destroy(). -/// Single instance is thread-safe and can be shared by many threads. Having -/// **multiple instances is safe but discouraged** as it has not benefits over -/// having the singleton. -/// -/// @param query_fn Pointer to query callback function. Nonnull. -/// @param update_fn Pointer to update callback function. Nonnull. -/// @param call_fn Pointer to call callback function. Nonnull. -/// @return Pointer to the created EVM instance. -typedef struct evm_instance* (*evm_create_fn)(evm_query_fn query_fn, - evm_update_fn update_fn, - evm_call_fn call_fn); - -/// Destroys the EVM instance. -/// -/// @param evm The EVM instance to be destroyed. -typedef void (*evm_destroy_fn)(struct evm_instance* evm); - - -/// Configures the EVM instance. -/// -/// Allows modifying options of the EVM instance. -/// Options: -/// - code cache behavior: on, off, read-only, ... -/// - optimizations, -/// -/// @param evm The EVM instance to be configured. -/// @param name The option name. Cannot be null. -/// @param value The new option value. Cannot be null. -/// @return 1 if the option set successfully, 0 otherwise. -typedef int (*evm_set_option_fn)(struct evm_instance* evm, - char const* name, - char const* value); - - -/// EVM compatibility mode aka chain mode. -/// TODO: Can you suggest better name? -enum evm_mode { - EVM_FRONTIER = 0, - EVM_HOMESTEAD = 1 -}; - - -/// Generates and executes machine code for given EVM bytecode. -/// -/// All the fun is here. This function actually does something useful. -/// -/// @param instance A EVM instance. -/// @param env A pointer to the execution environment provided by the -/// user and passed to callback functions. -/// @param mode EVM compatibility mode. -/// @param code_hash A hash of the bytecode, usually Keccak. The EVM uses it -/// as the code identifier. A EVM implementation is able to -/// hash the code itself if it requires it, but the host -/// application usually has the hash already. -/// @param code Reference to the bytecode to be executed. -/// @param code_size The length of the bytecode. -/// @param gas Gas for execution. Min 0, max 2^63-1. -/// @param input Reference to the input data. -/// @param input_size The size of the input data. -/// @param value Call value. -/// @return All execution results. -typedef struct evm_result (*evm_execute_fn)(struct evm_instance* instance, - struct evm_env* env, - enum evm_mode mode, - struct evm_hash256 code_hash, - uint8_t const* code, - size_t code_size, - int64_t gas, - uint8_t const* input, - size_t input_size, - struct evm_uint256 value); - -/// Releases resources assigned to an execution result. -/// -/// This function releases memory (and other resources, if any) assigned to the -/// specified execution result making the result object invalid. -/// -/// @param result The execution result which resource are to be released. The -/// result itself it not modified by this function, but becomes -/// invalid and user should discard it as well. -typedef void (*evm_release_result_fn)(struct evm_result const* result); - -/// Status of a code in VM. Useful for JIT-like implementations. -enum evm_code_status { - /// The code is uknown to the VM. - EVM_UNKNOWN, - - /// The code has been compiled and is available in memory. - EVM_READY, - - /// The compiled version of the code is available in on-disk cache. - EVM_CACHED, -}; - - -/// Get information the status of the code in the VM. -typedef enum evm_code_status -(*evm_get_code_status_fn)(struct evm_instance* instance, - enum evm_mode mode, - struct evm_hash256 code_hash); - -/// Request preparation of the code for faster execution. It is not required -/// to execute the code but allows compilation of the code ahead of time in -/// JIT-like VMs. -typedef void (*evm_prepare_code_fn)(struct evm_instance* instance, - enum evm_mode mode, - struct evm_hash256 code_hash, - uint8_t const* code, - size_t code_size); - -/// VM interface. -/// -/// Defines the implementation of EVM-C interface for a VM. -struct evm_interface { - /// EVM-C ABI version implemented by the VM. - /// - /// For future use to detect ABI incompatibilities. The EVM-C ABI version - /// represented by this file is in ::EVM_ABI_VERSION. - uint32_t abi_version; - - /// Pointer to function creating a VM's instance. - evm_create_fn create; - - /// Pointer to function destroying a VM's instance. - evm_destroy_fn destroy; - - /// Pointer to function execuing a code in a VM. - evm_execute_fn execute; - - /// Pointer to function releasing an execution result. - evm_release_result_fn release_result; - - /// Optional pointer to function returning a status of a code. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_get_code_status_fn get_code_status; - - /// Optional pointer to function compiling a code. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_prepare_code_fn prepare_code; - - /// Optional pointer to function modifying VM's options. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_set_option_fn set_option; -}; - -/// Example of a function exporting an interface for an example VM. -/// -/// Each VM implementation is obligates to provided a function returning -/// VM's interface. The function has to be named as `_get_interface()`. -/// -/// @return VM interface -struct evm_interface evmjit_get_interface(void); - - -// #if __cplusplus -// } -// #endif -// -// #endif // EVM_H -/// @} diff --git a/evmjit/__init__.py b/evmjit/__init__.py index ea19086..08cbfcf 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -1,55 +1,40 @@ -from _libevmjit import ffi, lib - - -def enum(**enums): - return type('Enum', (), enums) - -evm_mode = enum(EVM_FRONTIER=0, EVM_HOMESTEAD=1) - -evm_query_key = enum( - EVM_SLOAD = 0, #< Storage value of a given key for SLOAD. - EVM_ADDRESS = 1, #< Address of the contract for ADDRESS. - EVM_CALLER = 2, #< Message sender address for CALLER. - EVM_ORIGIN = 3, #< Transaction origin address for ORIGIN. - EVM_GAS_PRICE = 4, #< Transaction gas price for GASPRICE. - EVM_COINBASE = 5, #< Current block miner address for COINBASE. - EVM_DIFFICULTY = 6, #< Current block difficulty for DIFFICULTY. - EVM_GAS_LIMIT = 7, #< Current block gas limit for GASLIMIT. - EVM_NUMBER = 8, #< Current block number for NUMBER. - EVM_TIMESTAMP = 9, #< Current block timestamp for TIMESTAMP. - EVM_CODE_BY_ADDRESS = 10, #< Code by an address for EXTCODE/SIZE. - EVM_BALANCE = 11, #< Balance of a given address for BALANCE. - EVM_BLOCKHASH = 12 #< Block hash of by block number for BLOCKHASH. -) - - -def from_uint256(a): - # TODO: We could've used int.from_bytes here, but I don't know how to - # access bytes of uint256 - words = a.words - v = 0 - v = (v << 64) | words[3] - v = (v << 64) | words[2] - v = (v << 64) | words[1] - v = (v << 64) | words[0] - return v - - -def to_uint256(x): - """ Converts integer to EVM-C uint256.""" +from _evmjit import ffi, lib + + +def from_uint256be(uint256be): + """ Converts EVM-C uint256be to integer.""" + if hasattr(int, 'from_bytes'): # Python 3 + return int.from_bytes(uint256be.bytes, byteorder='big') + n = 0 + for i, b in enumerate(uint256be.bytes): + a = b << (31 - i) * 8 + n = n | a + return n + + +def to_uint256be(x): + """ Converts integer to EVM-C uint256be.""" assert x < 2**256 - words = [] - for i in range(4): - word = x & 0xffffffffffffffff - words.append(word) - x = x >> 64 - return (words, ) + + if hasattr(int, 'to_bytes'): # Python 3 + uint256be = x.to_bytes(256, byteorder='big') + else: + uint256be = '{:064x}'.format(x).decode('hex') + # Must be returned inside list or tuple to be converted to evm_uint256be + # struct by CFFI. + return (uint256be,) @ffi.def_extern() def evm_query(env, key, arg): - if key == evm_query_key.EVM_SLOAD: - arg = from_uint256(arg.uint256) + if key == EVMJIT.SLOAD: + arg = from_uint256be(arg.uint256be) + elif key in (EVMJIT.BALANCE, EVMJIT.CODE_BY_ADDRESS): + arg = ffi.buffer(arg.address.bytes) + elif key == EVMJIT.BLOCKHASH: + # FIXME: EVMJIT has a bug here. It passes int64 but does not check + # if the number is not greater. + arg = int(ffi.cast("uint64_t", arg.int64)) else: arg = None @@ -57,27 +42,30 @@ def evm_query(env, key, arg): res = env.query(key, arg) # Convert answer back to EVM-C. - if key in (evm_query_key.EVM_GAS_PRICE, - evm_query_key.EVM_DIFFICULTY, - evm_query_key.EVM_BALANCE, - evm_query_key.EVM_BLOCKHASH, - evm_query_key.EVM_SLOAD): - return {'uint256': to_uint256(res)} - - if key in (evm_query_key.EVM_ADDRESS, - evm_query_key.EVM_CALLER, - evm_query_key.EVM_ORIGIN, - evm_query_key.EVM_COINBASE): + if key in (EVMJIT.GAS_PRICE, + EVMJIT.DIFFICULTY, + EVMJIT.BALANCE, + EVMJIT.SLOAD): + print(key, type(res)) + return {'uint256be': to_uint256be(res)} + + if key in (EVMJIT.ADDRESS, + EVMJIT.CALLER, + EVMJIT.ORIGIN, + EVMJIT.COINBASE): assert len(res) == 20 - return {'address': res} + return {'address': (res,)} - if key in (evm_query_key.EVM_GAS_LIMIT, - evm_query_key.EVM_NUMBER, - evm_query_key.EVM_TIMESTAMP): + if key in (EVMJIT.GAS_LIMIT, + EVMJIT.NUMBER, + EVMJIT.TIMESTAMP): return {'int64': res} - if key == evm_query_key.EVM_CODE_BY_ADDRESS: - return {'data': res, 'data_size': len(res)} + if key == EVMJIT.BLOCKHASH: + return {'uint256be': (res,)} + + if key == EVMJIT.CODE_BY_ADDRESS: + return {'data': ffi.from_buffer(res), 'data_size': len(res)} @ffi.def_extern() @@ -85,9 +73,16 @@ def evm_update(env, key, arg1, arg2): env = ffi.from_handle(ffi.cast('void*', env)) # Preprocess arguments. - if key == lib.EVM_SSTORE: - arg1 = from_uint256(arg1.uint256) - arg2 = from_uint256(arg2.uint256) + if key == EVMJIT.SSTORE: + arg1 = from_uint256be(arg1.uint256be) + arg2 = from_uint256be(arg2.uint256be) + elif key == EVMJIT.LOG: + arg1 = ffi.buffer(arg1.data, arg1.data_size) + n_topics = arg2.data_size // 32 + arg2 = [ffi.buffer(arg2.data + (i * 32), 32) for i in range(n_topics)] + elif key == EVMJIT.SELFDESTRUCT: + arg1 = ffi.buffer(arg1.address.bytes) + arg2 = None env.update(key, arg1, arg2) @@ -95,8 +90,18 @@ def evm_update(env, key, arg1, arg2): @ffi.def_extern() def evm_call(env, kind, gas, address, value, input, input_size, output, output_size): + assert gas >= 0 and gas <= 2**64 - 1 env = ffi.from_handle(ffi.cast('void*', env)) - return env.call(kind) # FIXME + address = ffi.buffer(address.bytes) + value = from_uint256be(value) if kind != EVMJIT.DELEGATECALL else 0 + input = ffi.buffer(input, input_size) + result_code, out, gas_used = env.call(kind, gas, address, value, input) + if result_code != EVMJIT.SUCCESS: + gas_used |= lib.EVM_CALL_FAILURE + if out: + size = min(output_size, len(out)) + ffi.memmove(output, out, size) + return gas_used class Env(object): @@ -129,13 +134,48 @@ def gas_left(self): @property def output(self): - output_size = self.__res.output_size - if output_size == 0: - return b'' - return ffi.unpack(self.__res.output_data, self.__res.output_size) + """ Returns a copy of the output.""" + return ffi.buffer(self.__res.output_data, self.__res.output_size)[:] class EVMJIT: + # Execution compatibility mode + FRONTIER = lib.EVM_FRONTIER + HOMESTEAD = lib.EVM_HOMESTEAD + + # Query keys + SLOAD = lib.EVM_SLOAD + ADDRESS = lib.EVM_ADDRESS + CALLER = lib.EVM_CALLER + ORIGIN = lib.EVM_ORIGIN + GAS_PRICE = lib.EVM_GAS_PRICE + COINBASE = lib.EVM_COINBASE + DIFFICULTY = lib.EVM_DIFFICULTY + GAS_LIMIT = lib.EVM_GAS_LIMIT + NUMBER = lib.EVM_NUMBER + TIMESTAMP = lib.EVM_TIMESTAMP + CODE_BY_ADDRESS = lib.EVM_CODE_BY_ADDRESS + BALANCE = lib.EVM_BALANCE + BLOCKHASH = lib.EVM_BLOCKHASH + + # Update keys + SSTORE = lib.EVM_SSTORE + LOG = lib.EVM_LOG + SELFDESTRUCT = lib.EVM_SELFDESTRUCT + + # Result codes + SUCCESS = lib.EVM_SUCCESS + FAILURE = lib.EVM_FAILURE + + # Call kinds + CALL = lib.EVM_CALL + CALLCODE = lib.EVM_CALLCODE + DELEGATECALL = lib.EVM_DELEGATECALL + CREATE = lib.EVM_CREATE + + # TODO: The above constants comes from EVM-C and are not EVMJIT specific. + # Should we move them to EVM namespace? + def __init__(self): # Get virtual interface from EVMJIT module. self.interface = lib.evmjit_get_interface() @@ -160,7 +200,7 @@ def execute(self, env, mode, code_hash, code, gas, input, value): gas, input, len(input), - to_uint256(value)) + to_uint256be(value)) return Result(ret, self.interface.release_result) def set_option(self, name, value): diff --git a/evmjit/__main__.py b/evmjit/__main__.py deleted file mode 100644 index 2a5aa6e..0000000 --- a/evmjit/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import absolute_import - -from . import main - - -if __name__ == "__main__": - main() diff --git a/evmjit_build.py b/evmjit_build.py new file mode 100644 index 0000000..655dbb3 --- /dev/null +++ b/evmjit_build.py @@ -0,0 +1,124 @@ +import os +import subprocess +import tarfile +from os import path +from cffi import FFI +from io import BytesIO + +try: + from urllib2 import urlopen, URLError +except ImportError: + from urllib.request import urlopen + from urllib.error import URLError + +CDEF = """ +extern "Python" union evm_variant evm_query(struct evm_env* env, + enum evm_query_key key, + union evm_variant arg); +extern "Python" void evm_update(struct evm_env* env, + enum evm_update_key key, + union evm_variant arg1, + union evm_variant arg2); +extern "Python" int64_t evm_call(struct evm_env* env, + enum evm_call_kind kind, + int64_t gas, + struct evm_uint160be address, + struct evm_uint256be value, + uint8_t const* input, + size_t input_size, + uint8_t* output, + size_t output_size); + +struct evm_interface evmjit_get_interface(void); +""" + +EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/develop.tar.gz' +BUILD_DIR = path.join(path.abspath(path.dirname(__file__)), 'build') +EVMJIT_SRC_DIR = path.join(BUILD_DIR, 'evmjit', 'src') +EVMJIT_BUILD_DIR = path.join(BUILD_DIR, 'evmjit', 'build') +EVMJIT_INSTALL_DIR = path.join(BUILD_DIR, 'evmjit', 'install') + + +def download_tarball(url, outdir): + if (path.exists(outdir)): + return + basedir = path.dirname(outdir) + if (not path.exists(basedir)): + os.makedirs(basedir) + try: + r = urlopen(url) + if r.getcode() == 200: + content = BytesIO(r.read()) + content.seek(0) + with tarfile.open(fileobj=content) as tf: + dirname = tf.getnames()[0].partition('/')[0] + tf.extractall() + os.rename(dirname, outdir) + else: + raise SystemExit( + "Unable to download evmjit library: HTTP-Status: %d", + r.getcode() + ) + except URLError as ex: + raise SystemExit("Unable to download evmjit library: %s", + ex.message) + +if 'EVMJIT_INSTALL_PREFIX' in os.environ: + # Using prebuild version of EVMJIT. Good for development and testing. + prefix = os.environ['EVMJIT_INSTALL_PREFIX'] +else: + # Download and build EVMJIT. + download_tarball(EVMJIT_URL, EVMJIT_SRC_DIR) + if (not path.exists(EVMJIT_BUILD_DIR)): + os.makedirs(EVMJIT_BUILD_DIR) + + configure_cmd = [ + 'cmake', + '-DCMAKE_INSTALL_PREFIX={}'.format(EVMJIT_INSTALL_DIR), + '-DCMAKE_BUILD_TYPE=Release', + EVMJIT_SRC_DIR + ] + + build_cmd = [ + 'cmake', + '--build', EVMJIT_BUILD_DIR, + '--target', 'evmjit-standalone' + ] + if os.name == 'posix': + build_cmd += ['--', '-j16'] + + install_cmd = [ + 'cmake', + '--build', EVMJIT_BUILD_DIR, + '--target', 'install' + ] + + subprocess.check_call(' '.join(configure_cmd), shell=True, + cwd=EVMJIT_BUILD_DIR) + subprocess.check_call(' '.join(build_cmd), shell=True) + subprocess.check_call(' '.join(install_cmd), shell=True) + prefix = EVMJIT_INSTALL_DIR + + +# Basic configuration. +include_dir = path.join(prefix, 'include') +evm_header_file = path.join(include_dir, 'evm.h') +library_dir = path.join(prefix, 'lib') +libraries = ['evmjit-standalone', 'stdc++'] + +# Preprocess evm.h header. +# We want to extract only essential part stripping out preprocessor directives. +evm_header = open(evm_header_file).read() +evm_cdef_begin = evm_header.index('// BEGIN Python CFFI declarations') +evm_cdef_end = evm_header.index('// END Python CFFI declarations') +evm_cdef = evm_header[evm_cdef_begin:evm_cdef_end] + +ffibuilder = FFI() +ffibuilder.cdef(evm_cdef) +ffibuilder.cdef(CDEF) +ffibuilder.set_source('_evmjit', '#include ', + libraries=libraries, library_dirs=[library_dir], + include_dirs=[include_dir]) + +if __name__ == "__main__": + ffibuilder.compile() diff --git a/setup.py b/setup.py index a6a9ee4..b1f6851 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import errno import os import os.path @@ -29,7 +31,6 @@ sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, has_system_lib # Version of libevmjit to download if none exists in the `libevmjit` @@ -199,33 +200,29 @@ def run(self): setup( name="evmjit", - version="0.1.1", + version="0.10.0b1", description='FFI bindings to libevmjit', url='https://github.com/RomanZacharia/pyevmjit', - author='Roman Zacharia', + author=u'Roman Zacharia, Paweł Bylica', author_email='roman.zacharia@gmail.com', - license='MIT', - - setup_requires=['cffi>=1.3.0', 'pytest-runner==2.6.2'], - install_requires=['cffi>=1.3.0'], - tests_require=['pytest==2.8.7'], - - packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libevmjit')), - ext_package="evmjit", - cffi_modules=[ - "_cffi_build/build.py:ffi" - ], - - cmdclass={ - 'build_clib': build_clib, - 'build_ext': build_ext, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel - }, - distclass=Distribution, + license='MIT', # FIXME: Can we change to APACHE 2.0? + + setup_requires=['cffi>=1.8.2'], + install_requires=['cffi>=1.8.2'], + + packages=['evmjit'], + cffi_modules=['evmjit_build.py:ffibuilder'], + + # cmdclass={ + # 'build_clib': build_clib, + # 'build_ext': build_ext, + # 'develop': develop, + # 'egg_info': egg_info, + # 'sdist': sdist, + # 'bdist_wheel': bdist_wheel + # }, + # distclass=Distribution, zip_safe=False, classifiers=[ diff --git a/setup_support.py b/setup_support.py deleted file mode 100644 index f448af3..0000000 --- a/setup_support.py +++ /dev/null @@ -1,97 +0,0 @@ -import glob -import os -import shutil -from contextlib import contextmanager -from tempfile import mkdtemp - -import subprocess - - -@contextmanager -def workdir(): - cwd = os.getcwd() - tmpdir = mkdtemp() - os.chdir(tmpdir) - try: - yield - finally: - os.chdir(cwd) - shutil.rmtree(tmpdir) - - -@contextmanager -def redirect(stdchannel, dest_filename): - oldstdchannel = os.dup(stdchannel.fileno()) - dest_file = open(dest_filename, 'w') - os.dup2(dest_file.fileno(), stdchannel.fileno()) - try: - yield - finally: - if oldstdchannel is not None: - os.dup2(oldstdchannel, stdchannel.fileno()) - if dest_file is not None: - dest_file.close() - - -def absolute(*paths): - op = os.path - return op.realpath(op.abspath(op.join(op.dirname(__file__), *paths))) - - -def build_flags(library, type_, path): - """Return separated build flags from pkg-config output""" - - pkg_config_path = [path] - if "PKG_CONFIG_PATH" in os.environ: - pkg_config_path.append(os.environ['PKG_CONFIG_PATH']) - if "LIB_DIR" in os.environ: - pkg_config_path.append(os.environ['LIB_DIR']) - pkg_config_path.append(os.path.join(os.environ['LIB_DIR'], "pkgconfig")) - - options = [ - # "--static", - { - 'I': "--cflags-only-I", - 'L': "--libs-only-L", - 'l': "--libs-only-l" - }[type_] - ] - - return [ - flag.strip("-{}".format(type_)) - for flag - in subprocess.check_output( - ["pkg-config"] + options + [library], - env=dict(os.environ, PKG_CONFIG_PATH=":".join(pkg_config_path)) - ).decode("UTF-8").split() - ] - - -def _find_lib(): - from cffi import FFI - ffi = FFI() - try: - ffi.dlopen("evmjit") - except OSError: - if 'LIB_DIR' in os.environ: - for path in glob.glob(os.path.join(os.environ['LIB_DIR'], "*evmjit*")): - try: - FFI().dlopen(path) - return True - except OSError: - pass - # We couldn't locate libevmjit so we'll use the bundled one - return False - else: - # If we got this far then the system library should be good enough - return True - - -_has_system_lib = None - - -def has_system_lib(): - global _has_system_lib - if _has_system_lib is None: - _has_system_lib = _find_lib() - return _has_system_lib diff --git a/tests/test.py b/tests/test_evmjit.py similarity index 61% rename from tests/test.py rename to tests/test_evmjit.py index e941cec..f9461c3 100644 --- a/tests/test.py +++ b/tests/test_evmjit.py @@ -1,5 +1,5 @@ import hashlib -from evmjit import EVMJIT, Env, evm_mode, to_uint256 +from evmjit import EVMJIT, Env class TestEnv(Env): @@ -16,14 +16,6 @@ def call(self, kind, gas, address, value, input, input_size, output, return False -def test_to_uint256(): - assert to_uint256(0)[0] == [0, 0, 0, 0] - assert to_uint256(1)[0] == [1, 0, 0, 0] - assert to_uint256(1 << 63)[0] == [1 << 63, 0, 0, 0] - assert to_uint256(1 << 64)[0] == [0, 1, 0, 0] - assert to_uint256(1 << 255)[0] == [0, 0, 0, 1 << 63] - - def test_evm(): jit = EVMJIT() @@ -40,16 +32,16 @@ def test_evm(): success = jit.set_option("hello", "world") assert not success - ready = jit.is_code_ready(evm_mode.EVM_HOMESTEAD, code_hash) + ready = jit.is_code_ready(EVMJIT.HOMESTEAD, code_hash) assert not ready - jit.prepare_code(evm_mode.EVM_HOMESTEAD, code_hash, code) + jit.prepare_code(EVMJIT.HOMESTEAD, code_hash, code) - ready = jit.is_code_ready(evm_mode.EVM_HOMESTEAD, code_hash) + ready = jit.is_code_ready(EVMJIT.HOMESTEAD, code_hash) assert ready env = TestEnv() - result = jit.execute(env, evm_mode.EVM_HOMESTEAD, code_hash, code, gas, + result = jit.execute(env, EVMJIT.HOMESTEAD, code_hash, code, gas, input, value) assert result.code == 0 # Success. @@ -58,5 +50,4 @@ def test_evm(): if __name__ == "__main__": - test_to_uint256() test_evm() diff --git a/tests/test_json_vmtests.py b/tests/test_json_vmtests.py new file mode 100644 index 0000000..e45af32 --- /dev/null +++ b/tests/test_json_vmtests.py @@ -0,0 +1,126 @@ +import json +import os +from binascii import hexlify +from os import path + +import hashlib +import sha3 # Patches hashlib. # noqa + +from evmjit import EVMJIT + + +def code_hash(code): + h = hashlib.sha3_256() + h.update(code) + return h.digest() + + +class Env(object): + def __init__(self, desc): + self.desc = desc + self.addr = desc['exec']['address'].decode('hex') + self.caller = desc['exec']['caller'].decode('hex') + self.tx_origin = desc['exec']['origin'].decode('hex') + self.block_number = int(desc['env']['currentNumber'], 16) + self.block_timestamp = int(desc['env']['currentTimestamp'], 16) + self.block_gas_limit = int(desc['env']['currentGasLimit'], 16) + self.block_difficulty = int(desc['env']['currentDifficulty'], 16) + self.block_coinbase = desc['env']['currentCoinbase'].decode('hex') + self.storage = desc['pre'][hexlify(self.addr)]['storage'] + + def get_balance(self, addr): + addr = addr.encode('hex') + if addr not in self.pre: + return 0 + return int(self.pre[addr]['balance'], 16) + + def query(self, key, arg): + print("query(key: {}, arg: {})".format(key, arg)) + if key == EVMJIT.SLOAD: + sload_key = '{:x}'.format(arg) + value = self.pre[hexlify(self.addr)]['storage'].get(sload_key, '0') + return int(value, 16) + if key == EVMJIT.ADDRESS: + return self.addr + if key == EVMJIT.CALLER: + return self.caller + if key == EVMJIT.ORIGIN: + return self.tx_origin + if key == EVMJIT.COINBASE: + return self.block_coinbase + if key == EVMJIT.NUMBER: + return self.block_number + if key == EVMJIT.TIMESTAMP: + return self.block_timestamp + if key == EVMJIT.GAS_LIMIT: + return self.block_gas_limit + if key == EVMJIT.DIFFICULTY: + return self.block_difficulty + if key == EVMJIT.BLOCKHASH: + return b'\x01' * 32 + + def update(self, key, arg1, arg2): + print("update(key: {}, arg1: {}, arg2: {})".format(key, arg1, arg2)) + if key == EVMJIT.SSTORE: + key = '0x{:02x}'.format(arg1) + val = '0x{:02x}'.format(arg2) + self.storage[key] = val + + def call(self, kind, gas, address, value, input): + if kind != EVMJIT.DELEGATECALL: + if self.get_balance(self.addr) < value: + return EVMJIT.FAILURE, b'', 0 + if kind == EVMJIT.CREATE: + print("create(gas: {}, value: {}, code: {})".format( + gas, value, input)) + return EVMJIT.SUCCESS, b'', 0 + + if kind == EVMJIT.DELEGATECALL: + assert value == 0 + + cost = 0 + if value: + cost += 9000 - 2300 + + if hexlify(address) not in self.desc['pre']: + cost += 25000 + + print("call(kind: {}, gas: {}, value: {})".format(kind, gas, value)) + return EVMJIT.SUCCESS, b'', cost + + +def test_vmtests(): + jit = EVMJIT() + + tests_dir = os.environ['ETHEREUM_TEST_PATH'] + vmtests_dir = path.join(tests_dir, 'VMTests') + json_test_files = [] + for subdir, _, files in os.walk(vmtests_dir): + json_test_files += [path.join(subdir, f) + for f in files if f.endswith('.json')] + for json_test_file in json_test_files: + print(json_test_file) + test_suite = json.load(open(json_test_file)) + for name, desc in test_suite.iteritems(): + print(name) + assert name != 'pc1' + # pprint(desc) + ex = desc['exec'] + code = ex['code'][2:].decode('hex') + data = ex['data'][2:].decode('hex') + gas = int(ex['gas'], 16) + value = int(ex['value'], 16) + env = Env(desc) + res = jit.execute(env, EVMJIT.FRONTIER, code_hash(code), + code, gas, data, value) + if 'gas' in desc: + assert res.code == EVMJIT.SUCCESS + assert env.storage == desc['post'][hexlify(env.addr)]['storage'] + expected_gas = int(desc['gas'], 16) + assert res.gas_left == expected_gas + else: + assert res.code != EVMJIT.SUCCESS + assert False + +if __name__ == '__main__': + test_vmtests()