From 2eb507d1bc78a94d0733f21a8e36d1b02586cec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 31 Aug 2016 18:47:06 +0200 Subject: [PATCH 01/14] Simplify setup.py --- evmjit/__init__.py | 45 +++++++++++++++++++-------------------- evmjit_build.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 32 +++++++++++++--------------- tests/test.py | 11 +--------- 4 files changed, 88 insertions(+), 52 deletions(-) create mode 100644 evmjit_build.py diff --git a/evmjit/__init__.py b/evmjit/__init__.py index ea19086..54da968 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -1,4 +1,4 @@ -from _libevmjit import ffi, lib +from _evmjit import ffi, lib def enum(**enums): @@ -23,33 +23,30 @@ def enum(**enums): ) -def from_uint256(a): - # TODO: We could've used int.from_bytes here, but I don't know how to - # access bytes of uint256 - words = a.words - v = 0 - v = (v << 64) | words[3] - v = (v << 64) | words[2] - v = (v << 64) | words[1] - v = (v << 64) | words[0] - return v +def from_uint256be(uint256be): + """ Converts EVM-C uint256be to integer.""" + if hasattr(int, 'from_bytes'): # Python 3 + return int.from_bytes(uint256be.bytes, byteorder='big') + return int(bytes(uint256be.bytes).encode('hex'), 16) -def to_uint256(x): - """ Converts integer to EVM-C uint256.""" +def to_uint256be(x): + """ Converts integer to EVM-C uint256be.""" assert x < 2**256 - words = [] - for i in range(4): - word = x & 0xffffffffffffffff - words.append(word) - x = x >> 64 - return (words, ) + + if hasattr(int, 'to_bytes'): # Python 3 + uint256be = x.to_bytes(256, byteorder='big') + else: + uint256be = '{:064x}'.format(x).decode('hex') + # Must be returned inside list or tuple to be converted to evm_uint256be + # struct by CFFI. + return (uint256be,) @ffi.def_extern() def evm_query(env, key, arg): if key == evm_query_key.EVM_SLOAD: - arg = from_uint256(arg.uint256) + arg = from_uint256be(arg.uint256be) else: arg = None @@ -62,7 +59,7 @@ def evm_query(env, key, arg): evm_query_key.EVM_BALANCE, evm_query_key.EVM_BLOCKHASH, evm_query_key.EVM_SLOAD): - return {'uint256': to_uint256(res)} + return {'uint256be': to_uint256be(res)} if key in (evm_query_key.EVM_ADDRESS, evm_query_key.EVM_CALLER, @@ -86,8 +83,8 @@ def evm_update(env, key, arg1, arg2): # Preprocess arguments. if key == lib.EVM_SSTORE: - arg1 = from_uint256(arg1.uint256) - arg2 = from_uint256(arg2.uint256) + arg1 = from_uint256be(arg1.uint256be) + arg2 = from_uint256be(arg2.uint256be) env.update(key, arg1, arg2) @@ -160,7 +157,7 @@ def execute(self, env, mode, code_hash, code, gas, input, value): gas, input, len(input), - to_uint256(value)) + to_uint256be(value)) return Result(ret, self.interface.release_result) def set_option(self, name, value): diff --git a/evmjit_build.py b/evmjit_build.py new file mode 100644 index 0000000..6d175fd --- /dev/null +++ b/evmjit_build.py @@ -0,0 +1,52 @@ +import os +from os import path +from cffi import FFI + +CDEF = """ +extern "Python" union evm_variant evm_query(struct evm_env* env, + enum evm_query_key key, + union evm_variant arg); +extern "Python" void evm_update(struct evm_env* env, + enum evm_update_key key, + union evm_variant arg1, + union evm_variant arg2); +extern "Python" int64_t evm_call(struct evm_env* env, + enum evm_call_kind kind, + int64_t gas, + struct evm_uint160be address, + struct evm_uint256be value, + uint8_t const* input, + size_t input_size, + uint8_t* output, + size_t output_size); + +struct evm_interface evmjit_get_interface(void); +""" + + +if 'EVMJIT_INSTALL_PREFIX' in os.environ: + # Using prebuild version of EVMJIT. Good for development and testing. + prefix = os.environ['EVMJIT_INSTALL_PREFIX'] + +# Basic configuration. +include_dir = path.join(prefix, 'include') +evm_header_file = path.join(include_dir, 'evm.h') +library_dir = path.join(prefix, 'lib') +libraries = ['evmjit-standalone', 'stdc++'] + +# Preprocess evm.h header. +# We want to extract only essential part stripping out preprocessor directives. +evm_header = open(evm_header_file).read() +evm_cdef_begin = evm_header.index('// BEGIN Python CFFI declarations') +evm_cdef_end = evm_header.index('// END Python CFFI declarations') +evm_cdef = evm_header[evm_cdef_begin:evm_cdef_end] + +ffibuilder = FFI() +ffibuilder.cdef(evm_cdef) +ffibuilder.cdef(CDEF) +ffibuilder.set_source('_evmjit', '#include ', + libraries=libraries, library_dirs=[library_dir], + include_dirs=[include_dir]) + +if __name__ == "__main__": + ffibuilder.compile() diff --git a/setup.py b/setup.py index a6a9ee4..987d646 100644 --- a/setup.py +++ b/setup.py @@ -207,25 +207,21 @@ def run(self): author_email='roman.zacharia@gmail.com', license='MIT', - setup_requires=['cffi>=1.3.0', 'pytest-runner==2.6.2'], + setup_requires=['cffi>=1.3.0'], install_requires=['cffi>=1.3.0'], - tests_require=['pytest==2.8.7'], - - packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libevmjit')), - ext_package="evmjit", - cffi_modules=[ - "_cffi_build/build.py:ffi" - ], - - cmdclass={ - 'build_clib': build_clib, - 'build_ext': build_ext, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel - }, - distclass=Distribution, + + packages=['evmjit'], + cffi_modules=['evmjit_build.py:ffibuilder'], + + # cmdclass={ + # 'build_clib': build_clib, + # 'build_ext': build_ext, + # 'develop': develop, + # 'egg_info': egg_info, + # 'sdist': sdist, + # 'bdist_wheel': bdist_wheel + # }, + # distclass=Distribution, zip_safe=False, classifiers=[ diff --git a/tests/test.py b/tests/test.py index e941cec..fa581af 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,5 +1,5 @@ import hashlib -from evmjit import EVMJIT, Env, evm_mode, to_uint256 +from evmjit import EVMJIT, Env, evm_mode class TestEnv(Env): @@ -16,14 +16,6 @@ def call(self, kind, gas, address, value, input, input_size, output, return False -def test_to_uint256(): - assert to_uint256(0)[0] == [0, 0, 0, 0] - assert to_uint256(1)[0] == [1, 0, 0, 0] - assert to_uint256(1 << 63)[0] == [1 << 63, 0, 0, 0] - assert to_uint256(1 << 64)[0] == [0, 1, 0, 0] - assert to_uint256(1 << 255)[0] == [0, 0, 0, 1 << 63] - - def test_evm(): jit = EVMJIT() @@ -58,5 +50,4 @@ def test_evm(): if __name__ == "__main__": - test_to_uint256() test_evm() From cd5524016d7801b9d7f5c146e01d8fd44d9783ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 1 Sep 2016 13:56:28 +0200 Subject: [PATCH 02/14] Add bundled build of evmjit --- _cffi_build/build.py | 55 ------ _cffi_build/evm.h | 436 ------------------------------------------- evmjit/__main__.py | 7 - evmjit_build.py | 72 +++++++ setup.py | 1 - setup_support.py | 97 ---------- 6 files changed, 72 insertions(+), 596 deletions(-) delete mode 100755 _cffi_build/build.py delete mode 100755 _cffi_build/evm.h delete mode 100644 evmjit/__main__.py delete mode 100644 setup_support.py diff --git a/_cffi_build/build.py b/_cffi_build/build.py deleted file mode 100755 index aa427bc..0000000 --- a/_cffi_build/build.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import sys -from collections import namedtuple - -from cffi import FFI - -sys.path.append(os.path.abspath(os.path.dirname(__file__))) - - -def absolute(*paths): - op = os.path - return op.realpath(op.abspath(op.join(op.dirname(__file__), *paths))) - -Source = namedtuple('Source', ('h', 'include')) - - -def _mk_ffi(sources, name="_libevmjit", bundled=True, **kwargs): - ffi = FFI() - code = [] - if 'INCLUDE_DIR' in os.environ: - kwargs['include_dirs'] = [absolute(os.environ['INCLUDE_DIR'])] - if 'LIB_DIR' in os.environ: - kwargs['library_dirs'] = [absolute(os.environ['LIB_DIR'])] - for source in sources: - with open(source.h, 'rt') as h: - ffi.cdef(h.read()) - code.append(source.include) - if bundled: - code.append("#define PY_USE_BUNDLED") - ffi.set_source(name, "\n".join(code), **kwargs) - return ffi - - -_base = [Source(absolute("evm.h"), "#include \"_cffi_build/evm.h\"",)] -ffi = _mk_ffi(_base, libraries=['evmjit-standalone', 'stdc++']) -ffi.cdef(""" - extern "Python" union evm_variant evm_query(struct evm_env* env, - enum evm_query_key key, - union evm_variant arg); - extern "Python" void evm_update(struct evm_env* env, - enum evm_update_key key, - union evm_variant arg1, - union evm_variant arg2); - extern "Python" int64_t evm_call( - struct evm_env* env, - enum evm_call_kind kind, - int64_t gas, - struct evm_hash160 address, - struct evm_uint256 value, - uint8_t const* input, - size_t input_size, - uint8_t* output, - size_t output_size); -""") -ffi.compile() diff --git a/_cffi_build/evm.h b/_cffi_build/evm.h deleted file mode 100755 index 6fe302d..0000000 --- a/_cffi_build/evm.h +++ /dev/null @@ -1,436 +0,0 @@ -/// EVM-C -- C interface to Ethereum Virtual Machine -/// -/// ## High level design rules -/// 1. Pass function arguments and results by value. -/// This rule comes from modern C++ and tries to avoid costly alias analysis -/// needed for optimization. As the result we have a lots of complex structs -/// and unions. And variable sized arrays of bytes cannot be passed by copy. -/// 2. The EVM operates on integers so it prefers values to be host-endian. -/// On the other hand, LLVM can generate good code for byte swaping. -/// The interface also tries to match host application "natural" endianess. -/// I would like to know what endianess you use and where. -/// -/// @defgroup EVMC EVM-C -/// @{ -// #ifndef EVM_H -// #define EVM_H -// -// #include // Definition of int64_t, uint64_t. -// #include // Definition of size_t. -// -// #if __cplusplus -// extern "C" { -// #endif - -/// The EVM-C ABI version number matching the interface declared in this file. -static const uint32_t EVM_ABI_VERSION = 0; - -/// Host-endian 256-bit integer. -/// -/// 32 bytes of data representing host-endian (that means little-endian almost -/// all the time) 256-bit integer. This applies to the words[] order as well. -/// words[0] contains the 64 lowest precision bits, words[3] constains the 64 -/// highest precision bits. -struct evm_uint256 { - /// The 4 64-bit words of the integer. Memory aligned to 8 bytes. - uint64_t words[4]; -}; - -/// Big-endian 160-bit hash suitable for keeping an Ethereum address. -struct evm_hash160 { - /// The 20 bytes of the hash. - uint8_t bytes[20]; -}; - - -/// Big-endian 256-bit integer/hash. -/// -/// 32 bytes of data. For EVM that means big-endian 256-bit integer. Values of -/// this type are converted to host-endian values inside EVM. -struct evm_hash256 { - /// The 32 bytes of the integer/hash. - /// - /// The memory is expected be aligned to 8 bytes, but there is no portable - /// way to express that. - uint8_t bytes[32]; -}; - -/// The execution result code. -enum evm_result_code { - EVM_SUCCESS = 0, ///< Execution finished with success. - EVM_FAILURE = 1, ///< Generic execution failure. - EVM_OUT_OF_GAS = 2, - EVM_BAD_INSTRUCTION = 3, - EVM_BAD_JUMP_DESTINATION = 4, - EVM_STACK_OVERFLOW = 5, - EVM_STACK_UNDERFLOW = 6, -}; - -/// The EVM code execution result. -struct evm_result { - /// The execution result code. - enum evm_result_code code; - - /// The amount of gas left after the execution. - /// - /// The value is valid only if evm_result::code == ::EVM_SUCCESS. - int64_t gas_left; - - /// The reference to output data. The memory containing the output data - /// is owned by EVM and is freed with evm_release_result_fn(). - uint8_t const* output_data; - - /// The size of the output data. - size_t output_size; - - /// @name Optional - /// The optional information that EVM is not required to provide. - /// @{ - - /// The pointer to EVM-owned memory. For EVM internal use. - /// @see output_data. - void* internal_memory; - - /// The error message explaining the result code. - char const* error_message; - - /// @} -}; - -/// The query callback key. -enum evm_query_key { - EVM_SLOAD = 0, ///< Storage value of a given key for SLOAD. - EVM_ADDRESS = 1, ///< Address of the contract for ADDRESS. - EVM_CALLER = 2, ///< Message sender address for CALLER. - EVM_ORIGIN = 3, ///< Transaction origin address for ORIGIN. - EVM_GAS_PRICE = 4, ///< Transaction gas price for GASPRICE. - EVM_COINBASE = 5, ///< Current block miner address for COINBASE. - EVM_DIFFICULTY = 6, ///< Current block difficulty for DIFFICULTY. - EVM_GAS_LIMIT = 7, ///< Current block gas limit for GASLIMIT. - EVM_NUMBER = 8, ///< Current block number for NUMBER. - EVM_TIMESTAMP = 9, ///< Current block timestamp for TIMESTAMP. - EVM_CODE_BY_ADDRESS = 10, ///< Code by an address for EXTCODE/SIZE. - EVM_BALANCE = 11, ///< Balance of a given address for BALANCE. - EVM_BLOCKHASH = 12 ///< Block hash of by block number for BLOCKHASH. -}; - - -/// Opaque struct representing execution enviroment managed by the host -/// application. -struct evm_env; - -/// Variant type to represent possible types of values used in EVM. -/// -/// Type-safety is lost around the code that uses this type. We should have -/// complete set of unit tests covering all possible cases. -/// The size of the type is 64 bytes and should fit in single cache line. -union evm_variant { - /// A host-endian 64-bit integer. - int64_t int64; - - /// A host-endian 256-bit integer. - struct evm_uint256 uint256; - - /// A big-endian 256-bit integer/hash. - struct evm_hash256 hash256; - - struct { - /// Additional padding to align the evm_variant::address with lower - /// bytes of a full 256-bit hash. - uint8_t address_padding[12]; - - /// An Ethereum address. - struct evm_hash160 address; - }; - - /// A memory reference. - struct { - /// Pointer to the data. - uint8_t const* data; - - /// Size of the referenced memory/data. - size_t data_size; - }; -}; - -/// Query callback function. -/// -/// This callback function is used by the EVM to query the host application -/// about additional data required to execute EVM code. -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param key The kind of the query. See evm_query_key and details below. -/// @param arg Additional argument to the query. It has defined value only for -/// the subset of query keys. -/// -/// ## Types of queries -/// Key | Arg | Expected result -/// ----------------------| -------------------- | ---------------------------- -/// ::EVM_GAS_PRICE | | evm_variant::uint256 -/// ::EVM_ADDRESS | | evm_variant::address -/// ::EVM_CALLER | | evm_variant::address -/// ::EVM_ORIGIN | | evm_variant::address -/// ::EVM_COINBASE | | evm_variant::address -/// ::EVM_DIFFICULTY | | evm_variant::uint256 -/// ::EVM_GAS_LIMIT | | evm_variant::uint256 -/// ::EVM_NUMBER | | evm_variant::int64? -/// ::EVM_TIMESTAMP | | evm_variant::int64? -/// ::EVM_CODE_BY_ADDRESS | evm_variant::address | evm_variant::data -/// ::EVM_BALANCE | evm_variant::address | evm_variant::uint256 -/// ::EVM_BLOCKHASH | evm_variant::int64 | evm_variant::hash256 -/// ::EVM_SLOAD | evm_variant::uint256 | evm_variant::uint256? -typedef union evm_variant (*evm_query_fn)(struct evm_env* env, - enum evm_query_key key, - union evm_variant arg); - -/// The update callback key. -enum evm_update_key { - EVM_SSTORE = 0, ///< Update storage entry - EVM_LOG = 1, ///< Log. - EVM_SELFDESTRUCT = 2, ///< Mark contract as selfdestructed and set - /// beneficiary address. -}; - - -/// Update callback function. -/// -/// This callback function is used by the EVM to modify contract state in the -/// host application. -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param key The kind of the update. See evm_update_key and details below. -/// -/// ## Kinds of updates -/// -/// - ::EVM_SSTORE -/// @param arg1 evm_variant::uint256 The index of the storage entry. -/// @param arg2 evm_variant::uint256 The value to be stored. -/// -/// - ::EVM_LOG -/// @param arg1 evm_variant::data The log unindexed data. -/// @param arg2 evm_variant::data The log topics. The referenced data is an -/// array of evm_hash256[] of possible length -/// from 0 to 4. So the valid -/// evm_variant::data_size values are 0, 32, 64 -/// 92 and 128. -/// -/// - ::EVM_SELFDESTRUCT -/// @param arg1 evm_variant::address The beneficiary address. -typedef void (*evm_update_fn)(struct evm_env* env, - enum evm_update_key key, - union evm_variant arg1, - union evm_variant arg2); - -/// The kind of call-like instruction. -enum evm_call_kind { - EVM_CALL = 0, ///< Request CALL. - EVM_DELEGATECALL = 1, ///< Request DELEGATECALL. The value param ignored. - EVM_CALLCODE = 2, ///< Request CALLCODE. - EVM_CREATE = 3 ///< Request CREATE. Semantic of some params changes. -}; - -/// The flag indicating call failure in evm_call_fn(). -static const int64_t EVM_CALL_FAILURE = INT64_MIN; - -/// Pointer to the callback function supporting EVM calls. -/// -/// @param env Pointer to execution environment managed by the host -/// application. -/// @param kind The kind of call-like opcode requested. -/// @param gas The amount of gas for the call. -/// @param address The address of a contract to be called. Ignored in case -/// of CREATE. -/// @param value The value sent to the callee. The endowment in case of -/// CREATE. -/// @param input The call input data or the CREATE init code. -/// @param input_size The size of the input data. -/// @param output The reference to the memory where the call output is to -/// be copied. In case of CREATE, the memory is guaranteed -/// to be at least 20 bytes to hold the address of the -/// created contract. -/// @param output_data The size of the output data. In case of CREATE, expected -/// value is 20. -/// @return If non-negative - the amount of gas left, -/// If negative - an exception occurred during the call/create. -/// There is no need to set 0 address in the output in this case. -typedef int64_t (*evm_call_fn)( - struct evm_env* env, - enum evm_call_kind kind, - int64_t gas, - struct evm_hash160 address, - struct evm_uint256 value, - uint8_t const* input, - size_t input_size, - uint8_t* output, - size_t output_size); - - -/// Opaque type representing a EVM instance. -struct evm_instance; - -/// Creates new EVM instance. -/// -/// Creates new EVM instance. The instance must be destroyed in evm_destroy(). -/// Single instance is thread-safe and can be shared by many threads. Having -/// **multiple instances is safe but discouraged** as it has not benefits over -/// having the singleton. -/// -/// @param query_fn Pointer to query callback function. Nonnull. -/// @param update_fn Pointer to update callback function. Nonnull. -/// @param call_fn Pointer to call callback function. Nonnull. -/// @return Pointer to the created EVM instance. -typedef struct evm_instance* (*evm_create_fn)(evm_query_fn query_fn, - evm_update_fn update_fn, - evm_call_fn call_fn); - -/// Destroys the EVM instance. -/// -/// @param evm The EVM instance to be destroyed. -typedef void (*evm_destroy_fn)(struct evm_instance* evm); - - -/// Configures the EVM instance. -/// -/// Allows modifying options of the EVM instance. -/// Options: -/// - code cache behavior: on, off, read-only, ... -/// - optimizations, -/// -/// @param evm The EVM instance to be configured. -/// @param name The option name. Cannot be null. -/// @param value The new option value. Cannot be null. -/// @return 1 if the option set successfully, 0 otherwise. -typedef int (*evm_set_option_fn)(struct evm_instance* evm, - char const* name, - char const* value); - - -/// EVM compatibility mode aka chain mode. -/// TODO: Can you suggest better name? -enum evm_mode { - EVM_FRONTIER = 0, - EVM_HOMESTEAD = 1 -}; - - -/// Generates and executes machine code for given EVM bytecode. -/// -/// All the fun is here. This function actually does something useful. -/// -/// @param instance A EVM instance. -/// @param env A pointer to the execution environment provided by the -/// user and passed to callback functions. -/// @param mode EVM compatibility mode. -/// @param code_hash A hash of the bytecode, usually Keccak. The EVM uses it -/// as the code identifier. A EVM implementation is able to -/// hash the code itself if it requires it, but the host -/// application usually has the hash already. -/// @param code Reference to the bytecode to be executed. -/// @param code_size The length of the bytecode. -/// @param gas Gas for execution. Min 0, max 2^63-1. -/// @param input Reference to the input data. -/// @param input_size The size of the input data. -/// @param value Call value. -/// @return All execution results. -typedef struct evm_result (*evm_execute_fn)(struct evm_instance* instance, - struct evm_env* env, - enum evm_mode mode, - struct evm_hash256 code_hash, - uint8_t const* code, - size_t code_size, - int64_t gas, - uint8_t const* input, - size_t input_size, - struct evm_uint256 value); - -/// Releases resources assigned to an execution result. -/// -/// This function releases memory (and other resources, if any) assigned to the -/// specified execution result making the result object invalid. -/// -/// @param result The execution result which resource are to be released. The -/// result itself it not modified by this function, but becomes -/// invalid and user should discard it as well. -typedef void (*evm_release_result_fn)(struct evm_result const* result); - -/// Status of a code in VM. Useful for JIT-like implementations. -enum evm_code_status { - /// The code is uknown to the VM. - EVM_UNKNOWN, - - /// The code has been compiled and is available in memory. - EVM_READY, - - /// The compiled version of the code is available in on-disk cache. - EVM_CACHED, -}; - - -/// Get information the status of the code in the VM. -typedef enum evm_code_status -(*evm_get_code_status_fn)(struct evm_instance* instance, - enum evm_mode mode, - struct evm_hash256 code_hash); - -/// Request preparation of the code for faster execution. It is not required -/// to execute the code but allows compilation of the code ahead of time in -/// JIT-like VMs. -typedef void (*evm_prepare_code_fn)(struct evm_instance* instance, - enum evm_mode mode, - struct evm_hash256 code_hash, - uint8_t const* code, - size_t code_size); - -/// VM interface. -/// -/// Defines the implementation of EVM-C interface for a VM. -struct evm_interface { - /// EVM-C ABI version implemented by the VM. - /// - /// For future use to detect ABI incompatibilities. The EVM-C ABI version - /// represented by this file is in ::EVM_ABI_VERSION. - uint32_t abi_version; - - /// Pointer to function creating a VM's instance. - evm_create_fn create; - - /// Pointer to function destroying a VM's instance. - evm_destroy_fn destroy; - - /// Pointer to function execuing a code in a VM. - evm_execute_fn execute; - - /// Pointer to function releasing an execution result. - evm_release_result_fn release_result; - - /// Optional pointer to function returning a status of a code. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_get_code_status_fn get_code_status; - - /// Optional pointer to function compiling a code. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_prepare_code_fn prepare_code; - - /// Optional pointer to function modifying VM's options. - /// - /// If the VM does not support this feature the pointer can be NULL. - evm_set_option_fn set_option; -}; - -/// Example of a function exporting an interface for an example VM. -/// -/// Each VM implementation is obligates to provided a function returning -/// VM's interface. The function has to be named as `_get_interface()`. -/// -/// @return VM interface -struct evm_interface evmjit_get_interface(void); - - -// #if __cplusplus -// } -// #endif -// -// #endif // EVM_H -/// @} diff --git a/evmjit/__main__.py b/evmjit/__main__.py deleted file mode 100644 index 2a5aa6e..0000000 --- a/evmjit/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import absolute_import - -from . import main - - -if __name__ == "__main__": - main() diff --git a/evmjit_build.py b/evmjit_build.py index 6d175fd..655dbb3 100644 --- a/evmjit_build.py +++ b/evmjit_build.py @@ -1,6 +1,15 @@ import os +import subprocess +import tarfile from os import path from cffi import FFI +from io import BytesIO + +try: + from urllib2 import urlopen, URLError +except ImportError: + from urllib.request import urlopen + from urllib.error import URLError CDEF = """ extern "Python" union evm_variant evm_query(struct evm_env* env, @@ -23,10 +32,73 @@ struct evm_interface evmjit_get_interface(void); """ +EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/develop.tar.gz' +BUILD_DIR = path.join(path.abspath(path.dirname(__file__)), 'build') +EVMJIT_SRC_DIR = path.join(BUILD_DIR, 'evmjit', 'src') +EVMJIT_BUILD_DIR = path.join(BUILD_DIR, 'evmjit', 'build') +EVMJIT_INSTALL_DIR = path.join(BUILD_DIR, 'evmjit', 'install') + + +def download_tarball(url, outdir): + if (path.exists(outdir)): + return + basedir = path.dirname(outdir) + if (not path.exists(basedir)): + os.makedirs(basedir) + try: + r = urlopen(url) + if r.getcode() == 200: + content = BytesIO(r.read()) + content.seek(0) + with tarfile.open(fileobj=content) as tf: + dirname = tf.getnames()[0].partition('/')[0] + tf.extractall() + os.rename(dirname, outdir) + else: + raise SystemExit( + "Unable to download evmjit library: HTTP-Status: %d", + r.getcode() + ) + except URLError as ex: + raise SystemExit("Unable to download evmjit library: %s", + ex.message) if 'EVMJIT_INSTALL_PREFIX' in os.environ: # Using prebuild version of EVMJIT. Good for development and testing. prefix = os.environ['EVMJIT_INSTALL_PREFIX'] +else: + # Download and build EVMJIT. + download_tarball(EVMJIT_URL, EVMJIT_SRC_DIR) + if (not path.exists(EVMJIT_BUILD_DIR)): + os.makedirs(EVMJIT_BUILD_DIR) + + configure_cmd = [ + 'cmake', + '-DCMAKE_INSTALL_PREFIX={}'.format(EVMJIT_INSTALL_DIR), + '-DCMAKE_BUILD_TYPE=Release', + EVMJIT_SRC_DIR + ] + + build_cmd = [ + 'cmake', + '--build', EVMJIT_BUILD_DIR, + '--target', 'evmjit-standalone' + ] + if os.name == 'posix': + build_cmd += ['--', '-j16'] + + install_cmd = [ + 'cmake', + '--build', EVMJIT_BUILD_DIR, + '--target', 'install' + ] + + subprocess.check_call(' '.join(configure_cmd), shell=True, + cwd=EVMJIT_BUILD_DIR) + subprocess.check_call(' '.join(build_cmd), shell=True) + subprocess.check_call(' '.join(install_cmd), shell=True) + prefix = EVMJIT_INSTALL_DIR + # Basic configuration. include_dir = path.join(prefix, 'include') diff --git a/setup.py b/setup.py index 987d646..3cb8730 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, has_system_lib # Version of libevmjit to download if none exists in the `libevmjit` diff --git a/setup_support.py b/setup_support.py deleted file mode 100644 index f448af3..0000000 --- a/setup_support.py +++ /dev/null @@ -1,97 +0,0 @@ -import glob -import os -import shutil -from contextlib import contextmanager -from tempfile import mkdtemp - -import subprocess - - -@contextmanager -def workdir(): - cwd = os.getcwd() - tmpdir = mkdtemp() - os.chdir(tmpdir) - try: - yield - finally: - os.chdir(cwd) - shutil.rmtree(tmpdir) - - -@contextmanager -def redirect(stdchannel, dest_filename): - oldstdchannel = os.dup(stdchannel.fileno()) - dest_file = open(dest_filename, 'w') - os.dup2(dest_file.fileno(), stdchannel.fileno()) - try: - yield - finally: - if oldstdchannel is not None: - os.dup2(oldstdchannel, stdchannel.fileno()) - if dest_file is not None: - dest_file.close() - - -def absolute(*paths): - op = os.path - return op.realpath(op.abspath(op.join(op.dirname(__file__), *paths))) - - -def build_flags(library, type_, path): - """Return separated build flags from pkg-config output""" - - pkg_config_path = [path] - if "PKG_CONFIG_PATH" in os.environ: - pkg_config_path.append(os.environ['PKG_CONFIG_PATH']) - if "LIB_DIR" in os.environ: - pkg_config_path.append(os.environ['LIB_DIR']) - pkg_config_path.append(os.path.join(os.environ['LIB_DIR'], "pkgconfig")) - - options = [ - # "--static", - { - 'I': "--cflags-only-I", - 'L': "--libs-only-L", - 'l': "--libs-only-l" - }[type_] - ] - - return [ - flag.strip("-{}".format(type_)) - for flag - in subprocess.check_output( - ["pkg-config"] + options + [library], - env=dict(os.environ, PKG_CONFIG_PATH=":".join(pkg_config_path)) - ).decode("UTF-8").split() - ] - - -def _find_lib(): - from cffi import FFI - ffi = FFI() - try: - ffi.dlopen("evmjit") - except OSError: - if 'LIB_DIR' in os.environ: - for path in glob.glob(os.path.join(os.environ['LIB_DIR'], "*evmjit*")): - try: - FFI().dlopen(path) - return True - except OSError: - pass - # We couldn't locate libevmjit so we'll use the bundled one - return False - else: - # If we got this far then the system library should be good enough - return True - - -_has_system_lib = None - - -def has_system_lib(): - global _has_system_lib - if _has_system_lib is None: - _has_system_lib = _find_lib() - return _has_system_lib From 899859be8802dc18fad20a3fc692dfabd418c4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 1 Sep 2016 14:04:59 +0200 Subject: [PATCH 03/14] Fix Travis CI config --- .travis.yml | 95 +++++++++++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8eaa97..e54a3be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,9 @@ sudo: required language: python +dist: trusty python: - "2.7" - - "3.3" - - "3.4" - - "3.5" env: global: @@ -15,50 +13,52 @@ env: - INCLUDE_DIR=./libevmjit_ext/include:/opt/gcc-4.6.3/include/c++/4.6.3:/opt/gcc-4.6.3/include/c++/4.6.3/x86_64-unknown-linux-gnu - WHEELBUILDER_IMAGE=quay.io/pypa/manylinux1_x86_64 - PYPI_USERNAME=insert_pypi_upload_username_here - matrix: - - BUNDLED=0 - - BUNDLED=1 + # matrix: + # - BUNDLED=0 + # - BUNDLED=1 cache: + ccache: true directories: - $HOME/.cache/pip - $HOME/.cache/python-dl + - build/evmjit/build # Sigh... Until Travis issue #2312 is fixed Travis' osx image doesn't properly # support Python so we need to manually specify this matrix... -matrix: - include: - - os: osx - language: generic - osx_image: xcode7.1 - python: 2.7 - env: - - TRAVIS_PYTHON_VERSION=2.7 - - BUNDLED=1 - - os: osx - language: generic - osx_image: xcode7.1 - python: 3.4 - env: - - TRAVIS_PYTHON_VERSION=3.4 - - BUNDLED=1 - - os: osx - language: generic - osx_image: xcode7.1 - python: 3.5 - env: - - TRAVIS_PYTHON_VERSION=3.5 - - BUNDLED=1 - # Build `manylinux1` wheels - - os: linux - language: python - python: 3.5 - sudo: required - services: - - docker - env: - - BUNDLED=1 - - BUILD_LINUX_WHEELS=1 +# matrix: +# include: +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 2.7 +# env: +# - TRAVIS_PYTHON_VERSION=2.7 +# - BUNDLED=1 +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 3.4 +# env: +# - TRAVIS_PYTHON_VERSION=3.4 +# - BUNDLED=1 +# - os: osx +# language: generic +# osx_image: xcode7.1 +# python: 3.5 +# env: +# - TRAVIS_PYTHON_VERSION=3.5 +# - BUNDLED=1 +# # Build `manylinux1` wheels +# - os: linux +# language: python +# python: 3.5 +# sudo: required +# services: +# - docker +# env: +# - BUNDLED=1 +# - BUILD_LINUX_WHEELS=1 addons: apt: @@ -71,10 +71,11 @@ addons: - libgmp-dev - libatomic-ops-dev -before_install: - - source .travis/install.sh +# before_install: +# - source .travis/install.sh install: + - cmake --version - python setup.py install # This is a bit of a hack: @@ -86,13 +87,15 @@ install: # into producing correct paths. For that to work we need to re-rename the # source back to it's correct name. script: - - mv evmjit _evmjit - - coverage run --parallel --include="*/site-packages/*.egg/evmjit/*" -m py.test - - mv _evmjit evmjit - - coverage combine + # - mv evmjit _evmjit + # - coverage run --parallel --include="*/site-packages/*.egg/evmjit/*" -m py.test + # - mv _evmjit evmjit + # - coverage combine + - cd tests + - python test.py after_success: - - coveralls + # - coveralls deploy: provider: script From 399d2c0defa5987d7a58d009190c439783f09278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 1 Sep 2016 14:49:38 +0200 Subject: [PATCH 04/14] Update setup info --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3cb8730..9d0f41a 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import errno import os import os.path @@ -198,13 +200,13 @@ def run(self): setup( name="evmjit", - version="0.1.1", + version="0.10.0-rc.1", description='FFI bindings to libevmjit', url='https://github.com/RomanZacharia/pyevmjit', - author='Roman Zacharia', + author=u'Roman Zacharia, Paweł Bylica', author_email='roman.zacharia@gmail.com', - license='MIT', + license='MIT', # FIXME: Can we change to APACHE 2.0? setup_requires=['cffi>=1.3.0'], install_requires=['cffi>=1.3.0'], From 9813d73243a439584d4ed7d8a2f82530bbc322eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 2 Sep 2016 10:48:08 +0200 Subject: [PATCH 05/14] Rename test.py -> test_evmjit.py to be picked up by py.test --- tests/{test.py => test_evmjit.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test.py => test_evmjit.py} (100%) diff --git a/tests/test.py b/tests/test_evmjit.py similarity index 100% rename from tests/test.py rename to tests/test_evmjit.py From e658e908d7830cbb948dbac779576832065f9928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 2 Sep 2016 10:59:51 +0200 Subject: [PATCH 06/14] Move enum values to EVMJIT --- evmjit/__init__.py | 78 +++++++++++++++++++++++--------------------- tests/test_evmjit.py | 10 +++--- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index 54da968..b07a3c3 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -1,28 +1,6 @@ from _evmjit import ffi, lib -def enum(**enums): - return type('Enum', (), enums) - -evm_mode = enum(EVM_FRONTIER=0, EVM_HOMESTEAD=1) - -evm_query_key = enum( - EVM_SLOAD = 0, #< Storage value of a given key for SLOAD. - EVM_ADDRESS = 1, #< Address of the contract for ADDRESS. - EVM_CALLER = 2, #< Message sender address for CALLER. - EVM_ORIGIN = 3, #< Transaction origin address for ORIGIN. - EVM_GAS_PRICE = 4, #< Transaction gas price for GASPRICE. - EVM_COINBASE = 5, #< Current block miner address for COINBASE. - EVM_DIFFICULTY = 6, #< Current block difficulty for DIFFICULTY. - EVM_GAS_LIMIT = 7, #< Current block gas limit for GASLIMIT. - EVM_NUMBER = 8, #< Current block number for NUMBER. - EVM_TIMESTAMP = 9, #< Current block timestamp for TIMESTAMP. - EVM_CODE_BY_ADDRESS = 10, #< Code by an address for EXTCODE/SIZE. - EVM_BALANCE = 11, #< Balance of a given address for BALANCE. - EVM_BLOCKHASH = 12 #< Block hash of by block number for BLOCKHASH. -) - - def from_uint256be(uint256be): """ Converts EVM-C uint256be to integer.""" if hasattr(int, 'from_bytes'): # Python 3 @@ -45,7 +23,7 @@ def to_uint256be(x): @ffi.def_extern() def evm_query(env, key, arg): - if key == evm_query_key.EVM_SLOAD: + if key == EVMJIT.SLOAD: arg = from_uint256be(arg.uint256be) else: arg = None @@ -54,26 +32,26 @@ def evm_query(env, key, arg): res = env.query(key, arg) # Convert answer back to EVM-C. - if key in (evm_query_key.EVM_GAS_PRICE, - evm_query_key.EVM_DIFFICULTY, - evm_query_key.EVM_BALANCE, - evm_query_key.EVM_BLOCKHASH, - evm_query_key.EVM_SLOAD): + if key in (EVMJIT.GAS_PRICE, + EVMJIT.DIFFICULTY, + EVMJIT.BALANCE, + EVMJIT.BLOCKHASH, + EVMJIT.SLOAD): return {'uint256be': to_uint256be(res)} - if key in (evm_query_key.EVM_ADDRESS, - evm_query_key.EVM_CALLER, - evm_query_key.EVM_ORIGIN, - evm_query_key.EVM_COINBASE): + if key in (EVMJIT.ADDRESS, + EVMJIT.CALLER, + EVMJIT.ORIGIN, + EVMJIT.COINBASE): assert len(res) == 20 return {'address': res} - if key in (evm_query_key.EVM_GAS_LIMIT, - evm_query_key.EVM_NUMBER, - evm_query_key.EVM_TIMESTAMP): + if key in (EVMJIT.GAS_LIMIT, + EVMJIT.NUMBER, + EVMJIT.TIMESTAMP): return {'int64': res} - if key == evm_query_key.EVM_CODE_BY_ADDRESS: + if key == EVMJIT.CODE_BY_ADDRESS: return {'data': res, 'data_size': len(res)} @@ -82,7 +60,7 @@ def evm_update(env, key, arg1, arg2): env = ffi.from_handle(ffi.cast('void*', env)) # Preprocess arguments. - if key == lib.EVM_SSTORE: + if key == EVMJIT.SSTORE: arg1 = from_uint256be(arg1.uint256be) arg2 = from_uint256be(arg2.uint256be) @@ -133,6 +111,32 @@ def output(self): class EVMJIT: + # Execution compatibility mode + FRONTIER = lib.EVM_FRONTIER + HOMESTEAD = lib.EVM_HOMESTEAD + + # Query keys + SLOAD = lib.EVM_SLOAD + ADDRESS = lib.EVM_ADDRESS + CALLER = lib.EVM_CALLER + ORIGIN = lib.EVM_ORIGIN + GAS_PRICE = lib.EVM_GAS_PRICE + COINBASE = lib.EVM_COINBASE + DIFFICULTY = lib.EVM_DIFFICULTY + GAS_LIMIT = lib.EVM_GAS_LIMIT + NUMBER = lib.EVM_NUMBER + TIMESTAMP = lib.EVM_TIMESTAMP + CODE_BY_ADDRESS = lib.EVM_CODE_BY_ADDRESS + BALANCE = lib.EVM_BALANCE + BLOCKHASH = lib.EVM_BLOCKHASH + + # Update keys + SSTORE = lib.EVM_SSTORE + LOG = lib.EVM_LOG + SELFDESTRUCT = lib.EVM_SELFDESTRUCT + # TODO: The above constants comes from EVM-C and are not EVMJIT specific. + # Should we move them to EVM namespace? + def __init__(self): # Get virtual interface from EVMJIT module. self.interface = lib.evmjit_get_interface() diff --git a/tests/test_evmjit.py b/tests/test_evmjit.py index fa581af..f9461c3 100644 --- a/tests/test_evmjit.py +++ b/tests/test_evmjit.py @@ -1,5 +1,5 @@ import hashlib -from evmjit import EVMJIT, Env, evm_mode +from evmjit import EVMJIT, Env class TestEnv(Env): @@ -32,16 +32,16 @@ def test_evm(): success = jit.set_option("hello", "world") assert not success - ready = jit.is_code_ready(evm_mode.EVM_HOMESTEAD, code_hash) + ready = jit.is_code_ready(EVMJIT.HOMESTEAD, code_hash) assert not ready - jit.prepare_code(evm_mode.EVM_HOMESTEAD, code_hash, code) + jit.prepare_code(EVMJIT.HOMESTEAD, code_hash, code) - ready = jit.is_code_ready(evm_mode.EVM_HOMESTEAD, code_hash) + ready = jit.is_code_ready(EVMJIT.HOMESTEAD, code_hash) assert ready env = TestEnv() - result = jit.execute(env, evm_mode.EVM_HOMESTEAD, code_hash, code, gas, + result = jit.execute(env, EVMJIT.HOMESTEAD, code_hash, code, gas, input, value) assert result.code == 0 # Success. From 9ab0d5a7040e98a13145e82c0c6fc4fc3b77c32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 2 Sep 2016 16:23:17 +0200 Subject: [PATCH 07/14] Add unit-tests, pull fixed EVMJIT --- evmjit/__init__.py | 22 ++++++++++- evmjit_build.py | 2 +- tests/test_json_vmtests.py | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 tests/test_json_vmtests.py diff --git a/evmjit/__init__.py b/evmjit/__init__.py index b07a3c3..c157be5 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -70,8 +70,15 @@ def evm_update(env, key, arg1, arg2): @ffi.def_extern() def evm_call(env, kind, gas, address, value, input, input_size, output, output_size): + assert gas >= 0 and gas <= 2**64 - 1 env = ffi.from_handle(ffi.cast('void*', env)) - return env.call(kind) # FIXME + value = from_uint256be(value) + input = ffi.unpack(input, input_size) + result_code, output, gas_left = env.call(kind, gas, address, value, input) + assert gas_left <= gas + if result_code != EVMJIT.SUCCESS: + gas_left |= EVMJIT.CALL_FAILURE + return gas_left class Env(object): @@ -134,6 +141,19 @@ class EVMJIT: SSTORE = lib.EVM_SSTORE LOG = lib.EVM_LOG SELFDESTRUCT = lib.EVM_SELFDESTRUCT + + # Result codes + SUCCESS = lib.EVM_SUCCESS + + # Call kinds + CALL = lib.EVM_CALL + CALLCODE = lib.EVM_CALLCODE + DELEGATECALL = lib.EVM_DELEGATECALL + CREATE = lib.EVM_CREATE + + # Call exception flag. + CALL_FAILURE = lib.EVM_CALL_FAILURE + # TODO: The above constants comes from EVM-C and are not EVMJIT specific. # Should we move them to EVM namespace? diff --git a/evmjit_build.py b/evmjit_build.py index 655dbb3..4e79f1d 100644 --- a/evmjit_build.py +++ b/evmjit_build.py @@ -32,7 +32,7 @@ struct evm_interface evmjit_get_interface(void); """ -EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/develop.tar.gz' +EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/python.tar.gz' BUILD_DIR = path.join(path.abspath(path.dirname(__file__)), 'build') EVMJIT_SRC_DIR = path.join(BUILD_DIR, 'evmjit', 'src') EVMJIT_BUILD_DIR = path.join(BUILD_DIR, 'evmjit', 'build') diff --git a/tests/test_json_vmtests.py b/tests/test_json_vmtests.py new file mode 100644 index 0000000..7196954 --- /dev/null +++ b/tests/test_json_vmtests.py @@ -0,0 +1,79 @@ +import json +import os +from os import path +from pprint import pprint + +import hashlib +import sha3 # Patches hashlib. # noqa + +from evmjit import EVMJIT + + +def code_hash(code): + h = hashlib.sha3_256() + h.update(code) + return h.digest() + + +class Env(object): + def __init__(self, desc): + self.desc = desc + self.out_storage = {} + + def query(self, key, arg): + print("query(key: {}, arg: {})".format(key, arg)) + if key == EVMJIT.SLOAD: + sload_key = '{:x}'.format(arg) + value = self.desc['pre'][self.desc['exec']['address']]['storage'].get(sload_key, '0') + print("Value: {}".format(value)) + return int(value, 16) + if key == EVMJIT.ADDRESS: + return self.desc['exec']['address'].decode('hex') + + def update(self, key, arg1, arg2): + print("update(key: {}, arg1: {}, arg2: {})".format(key, arg1, arg2)) + self.out_storage[arg1] = arg2 + + def call(self, kind, gas, address, value, input): + if kind == EVMJIT.CREATE: + print("create(gas: {}, value: {}, code: {})".format( + gas, value, input)) + return EVMJIT.SUCCESS, b'', gas + + print("call(kind: {}, gas: {}, value: {})".format(kind, gas, value)) + return EVMJIT.SUCCESS, b'', gas + + +def test_vmtests(): + jit = EVMJIT() + + tests_dir = os.environ['ETHEREUM_TEST_PATH'] + vmtests_dir = path.join(tests_dir, 'VMTests') + json_test_files = [] + for subdir, _, files in os.walk(vmtests_dir): + json_test_files += [path.join(subdir, f) + for f in files if f.endswith('.json')] + # print(json_test_files) + test_suite = json.load(open(json_test_files[0])) + for name, desc in test_suite.iteritems(): + print(name) + # pprint(desc) + ex = desc['exec'] + code = ex['code'][2:].decode('hex') + # print(code.encode('hex')) + # code = '7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600052'.decode('hex') + data = ex['data'][2:].decode('hex') + gas = int(ex['gas'], 16) + value = int(ex['value'], 16) + res = jit.execute(Env(desc), EVMJIT.FRONTIER, code_hash(code), code, + gas, data, value) + if 'gas' in desc: + assert res.code == EVMJIT.SUCCESS + expected_gas = int(desc['gas'], 16) + assert res.gas_left == expected_gas + else: + assert res.code != EVMJIT.SUCCESS + + +if __name__ == '__main__': + test_vmtests() From 2038b79629e704ae4e1e3fdf8012cb7e80320a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 5 Sep 2016 14:00:58 +0200 Subject: [PATCH 08/14] Fixes, all VM tests passed --- evmjit/__init__.py | 24 +++++++++-------- tests/test_json_vmtests.py | 55 ++++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index c157be5..00bc350 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -5,7 +5,11 @@ def from_uint256be(uint256be): """ Converts EVM-C uint256be to integer.""" if hasattr(int, 'from_bytes'): # Python 3 return int.from_bytes(uint256be.bytes, byteorder='big') - return int(bytes(uint256be.bytes).encode('hex'), 16) + n = 0 + for i, b in enumerate(uint256be.bytes): + a = b << (31 - i) * 8 + n = n | a + return n def to_uint256be(x): @@ -44,7 +48,7 @@ def evm_query(env, key, arg): EVMJIT.ORIGIN, EVMJIT.COINBASE): assert len(res) == 20 - return {'address': res} + return {'address': (res,)} if key in (EVMJIT.GAS_LIMIT, EVMJIT.NUMBER, @@ -72,13 +76,13 @@ def evm_call(env, kind, gas, address, value, input, input_size, output, output_size): assert gas >= 0 and gas <= 2**64 - 1 env = ffi.from_handle(ffi.cast('void*', env)) + address = ffi.buffer(address.bytes) value = from_uint256be(value) - input = ffi.unpack(input, input_size) - result_code, output, gas_left = env.call(kind, gas, address, value, input) - assert gas_left <= gas + input = ffi.buffer(input, input_size) + result_code, output, gas_used = env.call(kind, gas, address, value, input) if result_code != EVMJIT.SUCCESS: - gas_left |= EVMJIT.CALL_FAILURE - return gas_left + gas_used |= lib.EVM_CALL_FAILURE + return gas_used class Env(object): @@ -114,7 +118,7 @@ def output(self): output_size = self.__res.output_size if output_size == 0: return b'' - return ffi.unpack(self.__res.output_data, self.__res.output_size) + return ffi.buffer(self.__res.output_data, self.__res.output_size) class EVMJIT: @@ -144,6 +148,7 @@ class EVMJIT: # Result codes SUCCESS = lib.EVM_SUCCESS + FAILURE = lib.EVM_FAILURE # Call kinds CALL = lib.EVM_CALL @@ -151,9 +156,6 @@ class EVMJIT: DELEGATECALL = lib.EVM_DELEGATECALL CREATE = lib.EVM_CREATE - # Call exception flag. - CALL_FAILURE = lib.EVM_CALL_FAILURE - # TODO: The above constants comes from EVM-C and are not EVMJIT specific. # Should we move them to EVM namespace? diff --git a/tests/test_json_vmtests.py b/tests/test_json_vmtests.py index 7196954..c2847ae 100644 --- a/tests/test_json_vmtests.py +++ b/tests/test_json_vmtests.py @@ -1,7 +1,7 @@ import json import os +from binascii import hexlify from os import path -from pprint import pprint import hashlib import sha3 # Patches hashlib. # noqa @@ -18,30 +18,71 @@ def code_hash(code): class Env(object): def __init__(self, desc): self.desc = desc + self.pre = desc['pre'] + self.addr = desc['exec']['address'].decode('hex') + self.caller = desc['exec']['caller'].decode('hex') + self.tx_origin = desc['exec']['origin'].decode('hex') + self.block_number = int(desc['env']['currentNumber'], 16) + self.block_timestamp = int(desc['env']['currentTimestamp'], 16) + self.block_gas_limit = int(desc['env']['currentGasLimit'], 16) + self.block_difficulty = int(desc['env']['currentDifficulty'], 16) + self.block_coinbase = desc['env']['currentCoinbase'].decode('hex') self.out_storage = {} + def get_balance(self, addr): + addr = addr.encode('hex') + if addr not in self.pre: + return 0 + return int(self.pre[addr]['balance'], 16) + def query(self, key, arg): print("query(key: {}, arg: {})".format(key, arg)) if key == EVMJIT.SLOAD: sload_key = '{:x}'.format(arg) - value = self.desc['pre'][self.desc['exec']['address']]['storage'].get(sload_key, '0') - print("Value: {}".format(value)) + value = self.pre[hexlify(self.addr)]['storage'].get(sload_key, '0') return int(value, 16) if key == EVMJIT.ADDRESS: - return self.desc['exec']['address'].decode('hex') + return self.addr + if key == EVMJIT.CALLER: + return self.caller + if key == EVMJIT.ORIGIN: + return self.tx_origin + if key == EVMJIT.COINBASE: + return self.block_coinbase + if key == EVMJIT.NUMBER: + return self.block_number + if key == EVMJIT.TIMESTAMP: + return self.block_timestamp + if key == EVMJIT.GAS_LIMIT: + return self.block_gas_limit + if key == EVMJIT.DIFFICULTY: + return self.block_difficulty def update(self, key, arg1, arg2): print("update(key: {}, arg1: {}, arg2: {})".format(key, arg1, arg2)) self.out_storage[arg1] = arg2 def call(self, kind, gas, address, value, input): + if kind != EVMJIT.DELEGATECALL: + if self.get_balance(self.addr) < value: + return EVMJIT.FAILURE, b'', 0 if kind == EVMJIT.CREATE: print("create(gas: {}, value: {}, code: {})".format( gas, value, input)) - return EVMJIT.SUCCESS, b'', gas + return EVMJIT.SUCCESS, b'', 0 + + if kind == EVMJIT.DELEGATECALL: + assert value == 0 + + cost = 0 + if value: + cost += 9000 - 2300 + + if hexlify(address) not in self.desc['pre']: + cost += 25000 print("call(kind: {}, gas: {}, value: {})".format(kind, gas, value)) - return EVMJIT.SUCCESS, b'', gas + return EVMJIT.SUCCESS, b'', cost def test_vmtests(): @@ -60,8 +101,6 @@ def test_vmtests(): # pprint(desc) ex = desc['exec'] code = ex['code'][2:].decode('hex') - # print(code.encode('hex')) - # code = '7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600052'.decode('hex') data = ex['data'][2:].decode('hex') gas = int(ex['gas'], 16) value = int(ex['value'], 16) From e0f7fa60d781ce2dafa5a0f02a12e985cabda95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 5 Sep 2016 14:06:55 +0200 Subject: [PATCH 09/14] Travis CI: change test command --- .travis.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index e54a3be..f31e77a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,18 +64,14 @@ addons: apt: packages: - git - - libtool - cmake - - pkg-config - - libffi-dev - - libgmp-dev - - libatomic-ops-dev -# before_install: -# - source .travis/install.sh +before_install: + - git clone --depth 1 https://github.com/ethereum/tests ~/ethereum-tests + - export ETHEREUM_TEST_PATH=~/ethereum-tests + - pip install pytest sha3 install: - - cmake --version - python setup.py install # This is a bit of a hack: @@ -91,8 +87,7 @@ script: # - coverage run --parallel --include="*/site-packages/*.egg/evmjit/*" -m py.test # - mv _evmjit evmjit # - coverage combine - - cd tests - - python test.py + - py.test after_success: # - coveralls From 69110a763e15827d4b15c1542cd6ef7a09b83938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Tue, 6 Sep 2016 13:53:08 +0200 Subject: [PATCH 10/14] More callbacks support --- evmjit/__init__.py | 25 +++++++++++++----- setup.py | 4 +-- tests/test_json_vmtests.py | 52 ++++++++++++++++++++++---------------- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index 00bc350..c9c2c70 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -29,6 +29,10 @@ def to_uint256be(x): def evm_query(env, key, arg): if key == EVMJIT.SLOAD: arg = from_uint256be(arg.uint256be) + elif key in (EVMJIT.BALANCE, EVMJIT.CODE_BY_ADDRESS): + arg = ffi.buffer(arg.address.bytes) + elif key == EVMJIT.BLOCKHASH: + arg = arg.int64 else: arg = None @@ -39,8 +43,8 @@ def evm_query(env, key, arg): if key in (EVMJIT.GAS_PRICE, EVMJIT.DIFFICULTY, EVMJIT.BALANCE, - EVMJIT.BLOCKHASH, EVMJIT.SLOAD): + print(key, type(res)) return {'uint256be': to_uint256be(res)} if key in (EVMJIT.ADDRESS, @@ -55,8 +59,11 @@ def evm_query(env, key, arg): EVMJIT.TIMESTAMP): return {'int64': res} + if key == EVMJIT.BLOCKHASH: + return {'uint256be': (res,)} + if key == EVMJIT.CODE_BY_ADDRESS: - return {'data': res, 'data_size': len(res)} + return {'data': ffi.from_buffer(res), 'data_size': len(res)} @ffi.def_extern() @@ -67,6 +74,9 @@ def evm_update(env, key, arg1, arg2): if key == EVMJIT.SSTORE: arg1 = from_uint256be(arg1.uint256be) arg2 = from_uint256be(arg2.uint256be) + elif key == EVMJIT.SELFDESTRUCT: + arg1 = ffi.buffer(arg1.address.bytes) + arg2 = None env.update(key, arg1, arg2) @@ -79,9 +89,12 @@ def evm_call(env, kind, gas, address, value, input, input_size, output, address = ffi.buffer(address.bytes) value = from_uint256be(value) input = ffi.buffer(input, input_size) - result_code, output, gas_used = env.call(kind, gas, address, value, input) + result_code, out, gas_used = env.call(kind, gas, address, value, input) if result_code != EVMJIT.SUCCESS: gas_used |= lib.EVM_CALL_FAILURE + if out: + size = min(output_size, len(out)) + ffi.memmove(output, out, size) return gas_used @@ -115,10 +128,8 @@ def gas_left(self): @property def output(self): - output_size = self.__res.output_size - if output_size == 0: - return b'' - return ffi.buffer(self.__res.output_data, self.__res.output_size) + """ Returns a copy of the output.""" + return ffi.buffer(self.__res.output_data, self.__res.output_size)[:] class EVMJIT: diff --git a/setup.py b/setup.py index 9d0f41a..c359bb8 100644 --- a/setup.py +++ b/setup.py @@ -208,8 +208,8 @@ def run(self): author_email='roman.zacharia@gmail.com', license='MIT', # FIXME: Can we change to APACHE 2.0? - setup_requires=['cffi>=1.3.0'], - install_requires=['cffi>=1.3.0'], + setup_requires=['cffi>=1.8.2'], + install_requires=['cffi>=1.8.2'], packages=['evmjit'], cffi_modules=['evmjit_build.py:ffibuilder'], diff --git a/tests/test_json_vmtests.py b/tests/test_json_vmtests.py index c2847ae..e45af32 100644 --- a/tests/test_json_vmtests.py +++ b/tests/test_json_vmtests.py @@ -18,7 +18,6 @@ def code_hash(code): class Env(object): def __init__(self, desc): self.desc = desc - self.pre = desc['pre'] self.addr = desc['exec']['address'].decode('hex') self.caller = desc['exec']['caller'].decode('hex') self.tx_origin = desc['exec']['origin'].decode('hex') @@ -27,7 +26,7 @@ def __init__(self, desc): self.block_gas_limit = int(desc['env']['currentGasLimit'], 16) self.block_difficulty = int(desc['env']['currentDifficulty'], 16) self.block_coinbase = desc['env']['currentCoinbase'].decode('hex') - self.out_storage = {} + self.storage = desc['pre'][hexlify(self.addr)]['storage'] def get_balance(self, addr): addr = addr.encode('hex') @@ -57,10 +56,15 @@ def query(self, key, arg): return self.block_gas_limit if key == EVMJIT.DIFFICULTY: return self.block_difficulty + if key == EVMJIT.BLOCKHASH: + return b'\x01' * 32 def update(self, key, arg1, arg2): print("update(key: {}, arg1: {}, arg2: {})".format(key, arg1, arg2)) - self.out_storage[arg1] = arg2 + if key == EVMJIT.SSTORE: + key = '0x{:02x}'.format(arg1) + val = '0x{:02x}'.format(arg2) + self.storage[key] = val def call(self, kind, gas, address, value, input): if kind != EVMJIT.DELEGATECALL: @@ -94,25 +98,29 @@ def test_vmtests(): for subdir, _, files in os.walk(vmtests_dir): json_test_files += [path.join(subdir, f) for f in files if f.endswith('.json')] - # print(json_test_files) - test_suite = json.load(open(json_test_files[0])) - for name, desc in test_suite.iteritems(): - print(name) - # pprint(desc) - ex = desc['exec'] - code = ex['code'][2:].decode('hex') - data = ex['data'][2:].decode('hex') - gas = int(ex['gas'], 16) - value = int(ex['value'], 16) - res = jit.execute(Env(desc), EVMJIT.FRONTIER, code_hash(code), code, - gas, data, value) - if 'gas' in desc: - assert res.code == EVMJIT.SUCCESS - expected_gas = int(desc['gas'], 16) - assert res.gas_left == expected_gas - else: - assert res.code != EVMJIT.SUCCESS - + for json_test_file in json_test_files: + print(json_test_file) + test_suite = json.load(open(json_test_file)) + for name, desc in test_suite.iteritems(): + print(name) + assert name != 'pc1' + # pprint(desc) + ex = desc['exec'] + code = ex['code'][2:].decode('hex') + data = ex['data'][2:].decode('hex') + gas = int(ex['gas'], 16) + value = int(ex['value'], 16) + env = Env(desc) + res = jit.execute(env, EVMJIT.FRONTIER, code_hash(code), + code, gas, data, value) + if 'gas' in desc: + assert res.code == EVMJIT.SUCCESS + assert env.storage == desc['post'][hexlify(env.addr)]['storage'] + expected_gas = int(desc['gas'], 16) + assert res.gas_left == expected_gas + else: + assert res.code != EVMJIT.SUCCESS + assert False if __name__ == '__main__': test_vmtests() From 227d0aa9c47f7d4b7aa1bee90f977d0265a5e9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Tue, 6 Sep 2016 15:00:27 +0200 Subject: [PATCH 11/14] Add support for LOG --- evmjit/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index c9c2c70..114bb59 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -74,6 +74,10 @@ def evm_update(env, key, arg1, arg2): if key == EVMJIT.SSTORE: arg1 = from_uint256be(arg1.uint256be) arg2 = from_uint256be(arg2.uint256be) + elif key == EVMJIT.LOG: + arg1 = ffi.buffer(arg1.data, arg1.data_size) + n_topics = arg2.data_size // 32 + arg2 = [ffi.buffer(arg2.data + (i * 32), 32) for i in range(n_topics)] elif key == EVMJIT.SELFDESTRUCT: arg1 = ffi.buffer(arg1.address.bytes) arg2 = None From 3265f8df0b6497d33349fc84584af50741d07bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 8 Sep 2016 12:26:19 +0200 Subject: [PATCH 12/14] Use develop branch of evmjit again --- evmjit_build.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evmjit_build.py b/evmjit_build.py index 4e79f1d..655dbb3 100644 --- a/evmjit_build.py +++ b/evmjit_build.py @@ -32,7 +32,7 @@ struct evm_interface evmjit_get_interface(void); """ -EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/python.tar.gz' +EVMJIT_URL = 'https://github.com/ethereum/evmjit/archive/develop.tar.gz' BUILD_DIR = path.join(path.abspath(path.dirname(__file__)), 'build') EVMJIT_SRC_DIR = path.join(BUILD_DIR, 'evmjit', 'src') EVMJIT_BUILD_DIR = path.join(BUILD_DIR, 'evmjit', 'build') diff --git a/setup.py b/setup.py index c359bb8..b1f6851 100644 --- a/setup.py +++ b/setup.py @@ -200,7 +200,7 @@ def run(self): setup( name="evmjit", - version="0.10.0-rc.1", + version="0.10.0b1", description='FFI bindings to libevmjit', url='https://github.com/RomanZacharia/pyevmjit', From 55c2f07e0ea60309409ff2c544d417bcdc2bbc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 8 Sep 2016 18:01:30 +0200 Subject: [PATCH 13/14] Set value to 0 for delegate call --- evmjit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index 114bb59..93ff9de 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -91,7 +91,7 @@ def evm_call(env, kind, gas, address, value, input, input_size, output, assert gas >= 0 and gas <= 2**64 - 1 env = ffi.from_handle(ffi.cast('void*', env)) address = ffi.buffer(address.bytes) - value = from_uint256be(value) + value = from_uint256be(value) if kind != EVMJIT.DELEGATECALL else 0 input = ffi.buffer(input, input_size) result_code, out, gas_used = env.call(kind, gas, address, value, input) if result_code != EVMJIT.SUCCESS: From 4f682bae98b00702a82b2144a966b2191a33b9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 9 Sep 2016 13:59:48 +0200 Subject: [PATCH 14/14] Fix issue with BLOCKHASH() argument --- evmjit/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evmjit/__init__.py b/evmjit/__init__.py index 93ff9de..08cbfcf 100644 --- a/evmjit/__init__.py +++ b/evmjit/__init__.py @@ -32,7 +32,9 @@ def evm_query(env, key, arg): elif key in (EVMJIT.BALANCE, EVMJIT.CODE_BY_ADDRESS): arg = ffi.buffer(arg.address.bytes) elif key == EVMJIT.BLOCKHASH: - arg = arg.int64 + # FIXME: EVMJIT has a bug here. It passes int64 but does not check + # if the number is not greater. + arg = int(ffi.cast("uint64_t", arg.int64)) else: arg = None