diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 205491fe..bd19d346 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -225,7 +225,7 @@ jobs: EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }} RBC_TESTS_FULL: TRUE run: | - mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k omnisci + mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k heavyai - name: Run rbc tests shell: bash -l {0} @@ -236,7 +236,7 @@ jobs: EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }} RBC_TESTS_FULL: TRUE run: | - mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k omnisci + mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k heavyai - name: Show Omniscidb conda logs on failure [conda] shell: bash -l {0} diff --git a/rbc/__init__.py b/rbc/__init__.py index 843770e0..eefa324d 100644 --- a/rbc/__init__.py +++ b/rbc/__init__.py @@ -1,7 +1,7 @@ # Expose a temporary prototype. It will be replaced by proper # implementation soon. from .remotejit import RemoteJIT # noqa: F401 -from .omniscidb import RemoteOmnisci # noqa: F401 +from .heavydb import RemoteHeavyDB # noqa: F401 from ._version import get_versions __version__ = get_versions()['version'] diff --git a/rbc/externals/omniscidb.py b/rbc/externals/heavydb.py similarity index 100% rename from rbc/externals/omniscidb.py rename to rbc/externals/heavydb.py diff --git a/rbc/heavyai/__init__.py b/rbc/heavyai/__init__.py new file mode 100644 index 00000000..015d143a --- /dev/null +++ b/rbc/heavyai/__init__.py @@ -0,0 +1,17 @@ +from .array import * # noqa: F401, F403 +from .column import * # noqa: F401, F403 +from .bytes import * # noqa: F401, F403 +from .metatype import * # noqa: F401, F403 +from .text_encoding import * # noqa: F401, F403 +from .pipeline import * # noqa: F401, F403 +from .column_list import * # noqa: F401, F403 +from .table_function_manager import * # noqa: F401, F403 + +from . import mathimpl as math # noqa: F401 +from . import npyimpl as np # noqa: F401 +from . import python_operators as operators # noqa: F401 + +# initialize the array api +from rbc.stdlib import array_api # noqa: F401 + +__all__ = [s for s in dir() if not s.startswith('_')] diff --git a/rbc/omnisci_backend/omnisci_array.py b/rbc/heavyai/array.py similarity index 95% rename from rbc/omnisci_backend/omnisci_array.py rename to rbc/heavyai/array.py index 1670c3a6..2bbbbffd 100644 --- a/rbc/omnisci_backend/omnisci_array.py +++ b/rbc/heavyai/array.py @@ -4,9 +4,9 @@ __all__ = ['ArrayPointer', 'Array', 'OmnisciArrayType'] from rbc import typesystem, errors -from .omnisci_buffer import (BufferPointer, Buffer, - OmnisciBufferType, - omnisci_buffer_constructor) +from .buffer import (BufferPointer, Buffer, + OmnisciBufferType, + omnisci_buffer_constructor) from numba.core import extending, types from typing import Union, TypeVar diff --git a/rbc/omnisci_backend/omnisci_buffer.py b/rbc/heavyai/buffer.py similarity index 99% rename from rbc/omnisci_backend/omnisci_buffer.py rename to rbc/heavyai/buffer.py index 9e4a7bf4..70a4251c 100644 --- a/rbc/omnisci_backend/omnisci_buffer.py +++ b/rbc/heavyai/buffer.py @@ -25,7 +25,7 @@ import operator from collections import defaultdict -from .omnisci_metatype import OmnisciMetaType +from .metatype import OmnisciMetaType from llvmlite import ir import numpy as np from rbc import typesystem, irutils @@ -195,7 +195,7 @@ def omnisci_buffer_constructor(context, builder, sig, args): alloc_fn = irutils.get_or_insert_function(builder.module, alloc_fnty, alloc_fn_name) ptr8 = builder.call(alloc_fn, [element_count, element_size]) # remember possible temporary allocations so that when leaving a - # UDF/UDTF, these will be deallocated, see omnisci_pipeline.py. + # UDF/UDTF, these will be deallocated, see pipeline.py. builder_buffers[builder].append(ptr8) ptr = builder.bitcast(ptr8, context.get_value_type(ptr_type)) diff --git a/rbc/omnisci_backend/omnisci_bytes.py b/rbc/heavyai/bytes.py similarity index 97% rename from rbc/omnisci_backend/omnisci_bytes.py rename to rbc/heavyai/bytes.py index 201c9e1f..d5758522 100644 --- a/rbc/omnisci_backend/omnisci_bytes.py +++ b/rbc/heavyai/bytes.py @@ -11,7 +11,7 @@ __all__ = ['BytesPointer', 'Bytes', 'OmnisciBytesType'] from rbc import typesystem -from .omnisci_buffer import ( +from .buffer import ( BufferPointer, Buffer, OmnisciBufferType, omnisci_buffer_constructor) from numba.core import types, extending diff --git a/rbc/omnisci_backend/omnisci_column.py b/rbc/heavyai/column.py similarity index 96% rename from rbc/omnisci_backend/omnisci_column.py rename to rbc/heavyai/column.py index 2f3f3246..f3fc7677 100644 --- a/rbc/omnisci_backend/omnisci_column.py +++ b/rbc/heavyai/column.py @@ -9,8 +9,8 @@ from llvmlite import ir from rbc import typesystem, irutils -from .omnisci_buffer import Buffer, OmnisciBufferType, BufferType -from .omnisci_column_list import OmnisciColumnListType +from .buffer import Buffer, OmnisciBufferType, BufferType +from .column_list import OmnisciColumnListType from rbc.targetinfo import TargetInfo from numba.core import extending, types diff --git a/rbc/omnisci_backend/omnisci_column_list.py b/rbc/heavyai/column_list.py similarity index 100% rename from rbc/omnisci_backend/omnisci_column_list.py rename to rbc/heavyai/column_list.py diff --git a/rbc/omnisci_backend/mathimpl.py b/rbc/heavyai/mathimpl.py similarity index 100% rename from rbc/omnisci_backend/mathimpl.py rename to rbc/heavyai/mathimpl.py diff --git a/rbc/omnisci_backend/omnisci_metatype.py b/rbc/heavyai/metatype.py similarity index 100% rename from rbc/omnisci_backend/omnisci_metatype.py rename to rbc/heavyai/metatype.py diff --git a/rbc/omnisci_backend/npyimpl.py b/rbc/heavyai/npyimpl.py similarity index 100% rename from rbc/omnisci_backend/npyimpl.py rename to rbc/heavyai/npyimpl.py diff --git a/rbc/omnisci_backend/omnisci_pipeline.py b/rbc/heavyai/pipeline.py similarity index 99% rename from rbc/omnisci_backend/omnisci_pipeline.py rename to rbc/heavyai/pipeline.py index be6d2784..4b42402c 100644 --- a/rbc/omnisci_backend/omnisci_pipeline.py +++ b/rbc/heavyai/pipeline.py @@ -1,7 +1,7 @@ import operator from rbc.errors import NumbaTypeError -from .omnisci_buffer import BufferMeta, free_all_other_buffers +from .buffer import BufferMeta, free_all_other_buffers from numba.core import ir, types from numba.core.compiler import CompilerBase, DefaultPassBuilder from numba.core.compiler_machinery import FunctionPass, register_pass diff --git a/rbc/omnisci_backend/python_operators.py b/rbc/heavyai/python_operators.py similarity index 99% rename from rbc/omnisci_backend/python_operators.py rename to rbc/heavyai/python_operators.py index 61432268..d36ed3e5 100644 --- a/rbc/omnisci_backend/python_operators.py +++ b/rbc/heavyai/python_operators.py @@ -1,6 +1,6 @@ import operator from llvmlite import ir -from .omnisci_array import ArrayPointer, Array +from .array import ArrayPointer, Array from rbc import typesystem from numba.core import extending, types diff --git a/rbc/omnisci_backend/omnisci_table_function_manager.py b/rbc/heavyai/table_function_manager.py similarity index 100% rename from rbc/omnisci_backend/omnisci_table_function_manager.py rename to rbc/heavyai/table_function_manager.py diff --git a/rbc/omnisci_backend/omnisci_text_encoding.py b/rbc/heavyai/text_encoding.py similarity index 90% rename from rbc/omnisci_backend/omnisci_text_encoding.py rename to rbc/heavyai/text_encoding.py index 7459907d..085f6b15 100644 --- a/rbc/omnisci_backend/omnisci_text_encoding.py +++ b/rbc/heavyai/text_encoding.py @@ -3,7 +3,7 @@ __all__ = ['OmnisciTextEncodingDictType', 'TextEncodingDict'] -from .omnisci_metatype import OmnisciMetaType +from .metatype import OmnisciMetaType from rbc import typesystem diff --git a/rbc/heavydb.py b/rbc/heavydb.py new file mode 100644 index 00000000..5e5de47b --- /dev/null +++ b/rbc/heavydb.py @@ -0,0 +1,1479 @@ +"""HeavyDB client config functions +""" + +import ast +import inspect +import os +import re +import warnings +import configparser +import numpy +from collections import defaultdict, namedtuple +from .remotejit import RemoteJIT, RemoteCallCapsule +from .thrift.utils import resolve_includes +from .thrift import Client as ThriftClient +from . import heavyai +from .heavyai import ( + OmnisciArrayType, OmnisciBytesType, OmnisciTextEncodingDictType, + OmnisciOutputColumnType, OmnisciColumnType, + OmnisciCompilerPipeline, OmnisciCursorType, + BufferMeta, OmnisciColumnListType, OmnisciTableFunctionManagerType) +from .targetinfo import TargetInfo +from .irtools import compile_to_LLVM +from .errors import ForbiddenNameError, OmnisciServerError +from .utils import parse_version, version_date +from . import ctools +from . import typesystem + + +def get_literal_return(func, verbose=False): + """Return a literal value from the last function return statement or + None. + """ + def _convert(node): + if isinstance(node, (int, float, complex, str, tuple, list, dict)): + return node + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Num): + return _convert(node.n) + if isinstance(node, ast.Name): + if node.id in func.__code__.co_names: + return func.__globals__[node.id] + args = func.__code__.co_varnames[:func.__code__.co_argcount] + args_with_default = args[len(args) - len(func.__defaults__):] + if node.id in args_with_default: + return func.__defaults__[ + list(args_with_default).index(node.id)] + raise NotImplementedError(f'literal value from name `{node.id}`') + if isinstance(node, ast.BinOp): + if isinstance(node.op, ast.Add): + return _convert(node.left) + _convert(node.right) + if isinstance(node.op, ast.Sub): + return _convert(node.left) - _convert(node.right) + if isinstance(node.op, ast.Mult): + return _convert(node.left) * _convert(node.right) + if isinstance(node.op, ast.Div): + return _convert(node.left) / _convert(node.right) + if isinstance(node.op, ast.FloorDiv): + return _convert(node.left) // _convert(node.right) + if isinstance(node.op, ast.Mod): + return _convert(node.left) % _convert(node.right) + if isinstance(node.op, ast.Pow): + return _convert(node.left) ** _convert(node.right) + raise NotImplementedError( + f'literal value from binary op `{node.op}`') + raise NotImplementedError(f'literal value from `{node}`') + source = '' + intent = None + for line in inspect.getsourcelines(func)[0]: + if line.lstrip().startswith('@'): + continue + if intent is None: + intent = len(line) - len(line.lstrip()) + source += line[intent:] + + tree = ast.parse(source) + last_node = tree.body[0].body[-1] + if isinstance(last_node, ast.Return): + try: + return _convert(last_node.value) + except Exception as msg: + if verbose: + print(f'get_literal_return: {msg}') + print(source) + + +def _global_heavydb(): + """Implements singleton of a global RemoteOmnisci instance. + """ + config = get_client_config() + remotedb = dict(heavyai=RemoteHeavyDB, + omnisci=RemoteOmnisci)[config['dbname']](**config) + while True: + yield remotedb + + +global_heavydb_singleton = _global_heavydb() # generator object + + +def is_available(_cache={}): + """Return version tuple and None if HeavyDB/OmnisciDB server is + accessible or recent enough. Otherwise return None and the reason + about unavailability. + """ + + if not _cache: + remotedb = next(global_heavydb_singleton) + try: + version = remotedb.version + except Exception as msg: + _cache['reason'] = f'failed to get {type(remotedb).__name__} version: {msg}' + else: + print(f' {type(remotedb).__name__} version {version}') + if version[:2] >= (4, 6): + _cache['version'] = version + else: + _cache['reason'] = ( + f'expected {type(remotedb).__name__} version 4.6 or greater, got {version}') + return _cache.get('version', ()), _cache.get('reason', '') + + +def get_client_config(**config): + """Retrieve the HeavyDB client configuration parameters from a client + home directory. + + Two HeavyDB brands (HEAVYDB_BRAND) are supported: heavyai and + omnisci. + + Note that here the client configurations parameters are those that + are used to configure the client software such as rbc or pymapd. + This is different from heavydb instance configuration described in + https://docs.heavy.ai/installation-and-configuration/config-parameters + that is used for configuring the heavydb server software. + + In Linux clients, the HeavyDB client configuration is read from + :code:`$HOME/.config/$HEAVYDB_BRAND/client.conf` + + In Windows clients, the configuration is read from + :code:`%UserProfile/.config/%HEAVYDB_BRAND%/client.conf` or + :code:`%AllUsersProfile/.config/%HEAVYDB_BRAND%/client.conf` + + When :code:`HEAVYDB_CLIENT_CONF` or :code:`OMNISCI_CLIENT_CONF` + environment variable is defined then the configuration is read + from the file specified in this variable. + + The configuration file must use configuration language similar to + one used in MS Windows INI files. For HeavyDB client + configuration, the file may contain, for instance:: + + [user] + name: + password: + + [server] + host: + port: + dbname: + + Parameters + ---------- + config : dict + Specify configuration parameters that override the parameters + from configuration file. + + Returns + ------- + config : dict + A dictionary of `user`, `password`, `host`, `port`, `dbname` and + other RemoteJIT options. + """ + caller_config = config + _config = dict(user='admin', password='HyperInteractive', + host='localhost', port=6274) + _config.update(**config) + config = _config + + conf_file = None + for brand, client_conf_env in [('heavyai', 'HEAVYDB_CLIENT_CONF'), + ('omnisci', 'OMNISCI_CLIENT_CONF')]: + conf_file = os.environ.get(client_conf_env, None) + if conf_file is not None and not os.path.isfile(conf_file): + print('rbc.omnisci.get_client_config:' + f' {client_conf_env}={conf_file!r}' + ' is not a file, ignoring.') + conf_file = None + if conf_file is None: + conf_file_base = os.path.join('.config', brand, 'client.conf') + for prefix_env in ['UserProfile', 'AllUsersProfile', 'HOME']: + prefix = os.environ.get(prefix_env, None) + if prefix is not None: + fn = os.path.join(prefix, conf_file_base) + if os.path.isfile(fn): + conf_file = fn + break + if conf_file is not None: + break + + if conf_file is not None: + conf = configparser.ConfigParser() + conf.read(conf_file) + + if 'user' in conf: + user = conf['user'] + if 'name' in user and 'name' not in caller_config: + config['user'] = user['name'] + if 'password' in user and 'password' not in caller_config: + config['password'] = user['password'] + + if 'server' in conf: + server = conf['server'] + if 'host' in server and 'host' not in caller_config: + config['host'] = server['host'] + if 'port' in server and 'port' not in caller_config: + config['port'] = int(server['port']) + if 'dbname' in server and 'dbname' not in caller_config: + config['dbname'] = server['dbname'] + + if 'rbc' in conf: + rbc = conf['rbc'] + for k in ['debug', 'use_host_target']: + if k in rbc and k not in caller_config: + config[k] = rbc.getboolean(k) + + if 'dbname' not in config: + version = get_heavydb_version(host=config['host'], port=config['port']) + if version is not None and version[:2] >= (6, 0): + if version[:3] == (6, 0, 0) and version_date(version) < 20220301: + # TODO: remove this if-block when heavydb 6.0 is released. + config['dbname'] = 'omnisci' + else: + config['dbname'] = 'heavyai' + else: + config['dbname'] = 'omnisci' + + return config + + +def is_udtf(sig): + """Check if signature is a table function signature. + """ + if sig[0].annotation().get('kind') == 'UDTF': + return True + for a in sig[1]: + if isinstance(a, (OmnisciOutputColumnType, OmnisciColumnType, + OmnisciColumnListType, OmnisciTableFunctionManagerType)): + return True + return False + + +def is_sizer(t): + """Check if type is a type of a sizer argument: + int32_t | sizer=... + """ + return t.is_int and t.bits == 32 and 'sizer' in t.annotation() + + +def get_sizer_enum(t): + """Return sizer enum value as defined by the omniscidb server. + """ + sizer = t.annotation()['sizer'] + sizer = output_buffer_sizer_map.get(sizer or None, sizer) + for shortname, fullname in output_buffer_sizer_map.items(): + if sizer == fullname: + return sizer + raise ValueError(f'unknown sizer value ({sizer}) in {t}') + + +output_buffer_sizer_map = dict( + ConstantParameter='kUserSpecifiedConstantParameter', + RowMultiplier='kUserSpecifiedRowMultiplier', + Constant='kConstant', + SpecifiedParameter='kTableFunctionSpecifiedParameter', + PreFlight='kPreFlightParameter') + +# Default sizer is RowMultiplier: +output_buffer_sizer_map[None] = output_buffer_sizer_map['RowMultiplier'] + +user_specified_output_buffer_sizers = { + 'kUserSpecifiedConstantParameter', 'kUserSpecifiedRowMultiplier', +} + + +def type_to_type_name(typ: typesystem.Type): + """Return typesystem.Type as DatumType name. + """ + styp = typ.tostring(use_annotation=False, use_name=False) + type_name = dict( + int8='TINYINT', + int16='SMALLINT', + int32='INT', + int64='BIGINT', + float32='FLOAT', + float64='DOUBLE', + ).get(styp) + if type_name is not None: + return type_name + raise NotImplementedError(f'converting `{styp}` to DatumType not supported') + + +def type_name_to_dtype(type_name): + """Return DatumType name as the corresponding numpy dtype. + """ + dtype = dict( + SMALLINT=numpy.int16, + INT=numpy.int32, + BIGINT=numpy.int64, + FLOAT=numpy.float32, + # DECIMAL=, + DOUBLE=numpy.float32, + STR=numpy.str0, + # TIME=, + # TIMESTAMP=, + # DATE=, + BOOL=numpy.bool8, + # INTERVAL_DAY_TIME=, + # INTERVAL_YEAR_MONTH=, + # POINT=, + # LINESTRING=, + # POLYGON=, + # MULTIPOLYGON=, + TINYINT=numpy.int8, + # GEOMETRY=, + # GEOGRAPHY= + ).get(type_name) + if dtype is not None: + return dtype + raise NotImplementedError( + f'convert DatumType `{type_name}` to numpy dtype') + + +class OmnisciQueryCapsule(RemoteCallCapsule): + + use_execute_cache = True + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r})' + + +def get_heavydb_version(host='localhost', port=6274, _cache={}): + """Acquires the version of heavydb server. + """ + if (host, port) in _cache: + return _cache[host, port] + thrift_content = ''' +exception TMapDException { + 1: string error_msg +} +service Omnisci { + string get_version() throws (1: TMapDException e) +} +''' + client = ThriftClient( + host=host, + port=port, + multiplexed=False, + thrift_content=thrift_content, + socket_timeout=60000) + try: + version = client(Omnisci=dict(get_version=()))['Omnisci']['get_version'] + except Exception as msg: + print(f'failed to get heavydb version[host={host}, port={port}]: {msg}') + version = None + else: + version = parse_version(version) + _cache[host, port] = version + return version + + +class RemoteHeavyDB(RemoteJIT): + + """Usage: + + .. code:: python + + omnisci = RemoteOmnisci(host=..., port=...) + + @omnisci('int(int, int)') + def add(a, b): + return a + b + + omnisci.register() + + Use pymapd, for instance, to make a SQL query `select add(c1, + c2) from table` + """ + multiplexed = False + mangle_prefix = '' + + typesystem_aliases = dict( + bool='bool8', + Array='OmnisciArrayType', + Bytes='OmnisciBytesType', + Cursor='OmnisciCursorType', + Column='OmnisciColumnType', + OutputColumn='OmnisciOutputColumnType', + RowMultiplier='int32|sizer=RowMultiplier', + ConstantParameter='int32|sizer=ConstantParameter', + SpecifiedParameter='int32|sizer=SpecifiedParameter', + Constant='int32|sizer=Constant', + PreFlight='int32|sizer=PreFlight', + ColumnList='OmnisciColumnListType', + TextEncodingDict='OmnisciTextEncodingDictType', + TableFunctionManager='OmnisciTableFunctionManagerType<>', + UDTF='int32|kind=UDTF' + ) + + remote_call_capsule_cls = OmnisciQueryCapsule + default_remote_call_hold = True + supports_local_caller = False + + def __init__(self, + user='admin', + password='HyperInteractive', + host='localhost', + port=6274, + dbname='omnisci', + **options): + self.user = user + self.password = password + self.dbname = dbname + + thrift_filename = os.path.join(os.path.dirname(__file__), + 'omnisci.thrift') + content = resolve_includes( + open(thrift_filename).read(), + [os.path.dirname(thrift_filename)]) + for p in ['common.', 'extension_functions.']: + content = content.replace(p, '') + self.thrift_content = content + + RemoteJIT.__init__(self, host=host, port=port, **options) + + self._version = None + self._thrift_client = None + self._session_id = None + self._targets = None + self.thrift_typemap = defaultdict(dict) + self._init_thrift_typemap() + self.has_cuda = None + self._null_values = dict() + + # An user-defined device-LLVM IR mapping. + self.user_defined_llvm_ir = {} + + def _init_thrift_typemap(self): + """Initialize thrift type map using client thrift configuration. + """ + typemap = self.thrift_typemap + for typename, typ in self.thrift_client.thrift.__dict__.items(): + if hasattr(typ, '_NAMES_TO_VALUES'): + for name, value in typ._NAMES_TO_VALUES.items(): + typemap[typename][name] = value + typemap[typename+'-inverse'][value] = name + + @property + def version(self): + if self._version is None: + version = self.thrift_call('get_version') + self._version = parse_version(version) + if self._version[:2] < (5, 6): + msg = (f'OmniSciDB server v.{version} is too old (expected v.5.6 or newer) ' + 'and some features might not be available.') + warnings.warn(msg, PendingDeprecationWarning) + return self._version + + @property + def session_id(self): + if self._session_id is None: + user = self.user + pw = self.password + dbname = self.dbname + self._session_id = self.thrift_call('connect', user, pw, dbname) + return self._session_id + + @property + def thrift_client(self): + if self._thrift_client is None: + self._thrift_client = self.client + return self._thrift_client + + def thrift_call(self, name, *args, **kwargs): + client = kwargs.get('client') + if client is None: + client = self.thrift_client + + if name == 'register_runtime_udf' and self.version[:2] >= (4, 9): + warnings.warn('Using register_runtime_udf is deprecated, ' + 'use register_runtime_extension_functions instead.') + session_id, signatures, device_ir_map = args + udfs = [] + sig_re = re.compile(r"\A(?P[\w\d_]+)\s+" + r"'(?P[\w\d_]+)[(](?P.*)[)]'\Z") + thrift = client.thrift + ext_arguments_map = self._get_ext_arguments_map() + for signature in signatures.splitlines(): + m = sig_re.match(signature) + if m is None: + raise RuntimeError( + 'Failed to parse signature %r' % (signature)) + name = m.group('name') + rtype = ext_arguments_map[m.group('rtype')] + atypes = [ext_arguments_map[a.strip()] + for a in m.group('atypes').split(',')] + udfs.append(thrift.TUserDefinedFunction( + name, atypes, rtype)) + return self.thrift_call('register_runtime_extension_functions', + session_id, udfs, [], device_ir_map) + + if self.debug: + msg = 'thrift_call %s%s' % (name, args) + if len(msg) > 200: + msg = msg[:180] + '...' + msg[-15:] + print(msg) + try: + return client(Omnisci={name: args})['Omnisci'][name] + except client.thrift.TMapDException as msg: + m = re.match(r'.*CalciteContextException.*(' + r'No match found for function signature.*' + r'|Cannot apply.*)', + msg.error_msg) + if m: + raise OmnisciServerError(f'[Calcite] {m.group(1)}') + m = re.match(r'.*Exception: (.*)', msg.error_msg) + if m: + raise OmnisciServerError(f'{m.group(1)}') + m = re.match( + r'(.*)\: No match found for function signature (.*)\(.*\)', + msg.error_msg + ) + if m: + msg = (f"Undefined function call {m.group(2)!r} in" + f" SQL statement: {m.group(1)}") + raise OmnisciServerError(msg) + m = re.match(r'.*SQL Error: (.*)', msg.error_msg) + if m: + raise OmnisciServerError(f'{m.group(1)}') + m = re.match(r'Could not bind *', msg.error_msg) + if m: + raise OmnisciServerError(msg.error_msg) + m = re.match(r'Runtime extension functions registration is disabled.', + msg.error_msg) + if m: + msg = (f"{msg.error_msg} Please use server options --enable-runtime-udf" + " and/or --enable-table-functions") + raise OmnisciServerError(msg) + + # TODO: catch more known server failures here. + raise + + def get_tables(self): + """Return a list of table names stored in the OmnisciDB server. + """ + return self.thrift_call('get_tables', self.session_id) + + def get_table_details(self, table_name): + """Return details about OmnisciDB table. + """ + return self.thrift_call('get_table_details', + self.session_id, table_name) + + def load_table_columnar(self, table_name, **columnar_data): + """Load columnar data to OmnisciDB table. + + Warning: when connected to OmnisciDB < 5.3, the data is loaded + row-wise that will be very slow for large data sets. + + Parameters + ---------- + table_name : str + The name of table into the data is loaded. The table must exist. + + columnar_data : dict + A dictionary of column names and the corresponding column + values. A column values is a sequence of values. + The table must have the specified columns defined. + + """ + if not self._null_values: + self.retrieve_targets() # initializes null values + + int_col_types = ['TINYINT', 'SMALLINT', 'INT', 'BIGINT', 'BOOL', + 'DECIMAL', 'TIME', 'TIMESTAMP', 'DATE'] + real_col_types = ['FLOAT', 'DOUBLE'] + str_col_types = ['STR', 'POINT', 'LINESTRING', 'POLYGON', + 'MULTIPOLYGON'] + + datumtype_map = { + 'BOOL': 'boolean8', + 'TINYINT': 'int8', + 'SMALLINT': 'int16', + 'INT': 'int32', + 'BIGINT': 'int64', + 'FLOAT': 'float32', + 'DOUBLE': 'float64', + 'BOOL[]': 'Array', + 'TINYINT[]': 'Array', + 'SMALLINT[]': 'Array', + 'INT[]': 'Array', + 'BIGINT[]': 'Array', + 'FLOAT[]': 'Array', + 'DOUBLE[]': 'Array', + } + + thrift = self.thrift_client.thrift + table_details = self.get_table_details(table_name) + use_sql = self.version[:2] < (5, 3) + if not use_sql and self.version[:2] < (5, 5): + for column_data in columnar_data.values(): + for v in column_data: + if v is None: + break + elif isinstance(v, (tuple, list)): + for _v in v: + if _v is None: + break + else: + continue + break + else: + continue + use_sql = True + break + if use_sql: + rows = None + for column_name, column_data in columnar_data.items(): + if rows is None: + rows = [[None for j in range(len(columnar_data))] + for i in range(len(column_data))] + col_index = None + for i, ct in enumerate(table_details.row_desc): + if ct.col_name == column_name: + typeinfo = ct.col_type + col_index = i + datumtype = thrift.TDatumType._VALUES_TO_NAMES[ + typeinfo.type] + is_array = typeinfo.is_array + break + assert col_index is not None + for i, v in enumerate(column_data): + if v is None: + v = "NULL" + elif is_array: + if datumtype == 'BOOL': + v = ["'true'" if v_ else ("'false'" + if v_ is not None else "NULL") + for v_ in v] + else: + v = [(str(v_) if v_ is not None else "NULL") for v_ in v] + v = ', '.join(v) + v = f'ARRAY[{v}]' + else: + if datumtype == 'BOOL': + v = "'true'" if v else "'false'" + rows[i][col_index] = v + + for row in rows: + table_row = ', '.join(map(str, row)) + self.sql_execute( + f'INSERT INTO {table_name} VALUES ({table_row})') + return + + columns = [] + for column_name, column_data in columnar_data.items(): + typeinfo = None + for ct in table_details.row_desc: + if ct.col_name == column_name: + typeinfo = ct.col_type + break + if typeinfo is None: + raise ValueError( + f'OmnisciDB `{table_name}` has no column `{column_name}`') + datumtype = thrift.TDatumType._VALUES_TO_NAMES[typeinfo.type] + nulls = [v is None for v in column_data] + if typeinfo.is_array: + arrays = [] + for arr in column_data: + if arr is None: + arr_nulls = None + else: + arr_nulls = [v is None for v in arr] + if True in arr_nulls: + # TODO: null support for text + null_value = self._null_values[datumtype_map[datumtype]] + arr = [(null_value if v is None else v) for v in arr] + if datumtype in int_col_types: + arr_data = thrift.TColumnData(int_col=arr) + elif datumtype in real_col_types: + arr_data = thrift.TColumnData(real_col=arr) + elif datumtype in str_col_types: + arr_data = thrift.TColumnData(str_col=arr) + else: + raise NotImplementedError( + f'loading {datumtype} array data') + arrays.append(thrift.TColumn(data=arr_data, nulls=arr_nulls)) + col_data = thrift.TColumnData(arr_col=arrays) + else: + if True in nulls: + # TODO: null support for text + null_value = self._null_values[datumtype_map[datumtype]] + column_data = [(null_value if v is None else v) for v in column_data] + if datumtype in int_col_types: + col_data = thrift.TColumnData(int_col=column_data) + elif datumtype in real_col_types: + col_data = thrift.TColumnData(real_col=column_data) + elif datumtype in str_col_types: + col_data = thrift.TColumnData(str_col=column_data) + else: + raise NotImplementedError(f'loading {datumtype} data') + columns.append(thrift.TColumn(data=col_data, nulls=nulls)) + + self.thrift_call('load_table_binary_columnar', + self.session_id, table_name, columns) + + def _make_row_results_set(self, data): + # The following code is a stripped copy from omnisci/pymapd + + _typeattr = { + 'SMALLINT': 'int', + 'INT': 'int', + 'BIGINT': 'int', + 'TIME': 'int', + 'TIMESTAMP': 'int', + 'DATE': 'int', + 'BOOL': 'int', + 'FLOAT': 'real', + 'DECIMAL': 'real', + 'DOUBLE': 'real', + 'STR': 'str', + 'POINT': 'str', + 'LINESTRING': 'str', + 'POLYGON': 'str', + 'MULTIPOLYGON': 'str', + 'TINYINT': 'int', + 'GEOMETRY': 'str', + 'GEOGRAPHY': 'str', + } + + thrift = self.thrift_client.thrift + + def extract_col_vals(desc, val): + typename = thrift.TDatumType._VALUES_TO_NAMES[desc.col_type.type] + nulls = val.nulls + if hasattr(val.data, 'arr_col') and val.data.arr_col: + vals = [ + None if null else [ + None if _n else _v + for _n, _v in zip( + v.nulls, getattr(v.data, _typeattr[typename] + '_col'))] + for null, v in zip(nulls, val.data.arr_col) + ] + else: + vals = getattr(val.data, _typeattr[typename] + '_col') + vals = [None if null else v for null, v in zip(nulls, vals)] + return vals + + if data.row_set.columns: + nrows = len(data.row_set.columns[0].nulls) + ncols = len(data.row_set.row_desc) + columns = [ + extract_col_vals(desc, col) + for desc, col in zip(data.row_set.row_desc, data.row_set.columns) + ] + for i in range(nrows): + yield tuple(columns[j][i] for j in range(ncols)) + + def query_requires_register(self, query): + """Check if given query requires registration step. + """ + names = '|'.join(self.get_pending_names()) + return names and re.search(r'\b(' + names + r')\b', query, re.I) is not None + + def sql_execute(self, query): + """Execute SQL query in OmnisciDB server. + + Parameters + ---------- + query : str + SQL query string containing exactly one query. Multiple + queries are not supported. + + Returns + ------- + descr : object + Row description object + results : iterator + Iterator over rows. + + """ + if self.query_requires_register(query): + self.register() + columnar = True + if self.debug: + print(' %s;' % (query)) + result = self.thrift_call( + 'sql_execute', self.session_id, query, columnar, "", -1, -1) + + type_code_to_type_name = self.thrift_typemap['TDatumType-inverse'] + Description = namedtuple("Description", ["name", "type_name", "null_ok"]) + descr = [] + for col in result.row_set.row_desc: + descr.append(Description(col.col_name, + type_code_to_type_name[col.col_type.type], + col.col_type.nullable)) + return descr, self._make_row_results_set(result) + + _ext_arguments_map = None + + def _get_ext_arguments_map(self): + if self._ext_arguments_map is not None: + return self._ext_arguments_map + thrift = self.thrift_client.thrift + typemap = self.thrift_typemap + ext_arguments_map = { + 'int8': typemap['TExtArgumentType']['Int8'], + 'int16': typemap['TExtArgumentType']['Int16'], + 'int32': typemap['TExtArgumentType']['Int32'], + 'int64': typemap['TExtArgumentType']['Int64'], + 'float32': typemap['TExtArgumentType']['Float'], + 'float64': typemap['TExtArgumentType']['Double'], + 'int8*': typemap['TExtArgumentType']['PInt8'], + 'int16*': typemap['TExtArgumentType']['PInt16'], + 'int32*': typemap['TExtArgumentType']['PInt32'], + 'int64*': typemap['TExtArgumentType']['PInt64'], + 'float32*': typemap['TExtArgumentType']['PFloat'], + 'float64*': typemap['TExtArgumentType']['PDouble'], + 'bool': typemap['TExtArgumentType']['Bool'], + 'bool*': typemap['TExtArgumentType'].get('PBool'), + 'Array': typemap['TExtArgumentType'].get( + 'ArrayBool', typemap['TExtArgumentType']['ArrayInt8']), + 'Array': typemap['TExtArgumentType']['ArrayInt8'], + 'Array': typemap['TExtArgumentType']['ArrayInt16'], + 'Array': typemap['TExtArgumentType']['ArrayInt32'], + 'Array': typemap['TExtArgumentType']['ArrayInt64'], + 'Array': typemap['TExtArgumentType']['ArrayFloat'], + 'Array': typemap['TExtArgumentType']['ArrayDouble'], + 'Column': typemap['TExtArgumentType'].get('ColumnBool'), + 'Column': typemap['TExtArgumentType'].get('ColumnInt8'), + 'Column': typemap['TExtArgumentType'].get('ColumnInt16'), + 'Column': typemap['TExtArgumentType'].get('ColumnInt32'), + 'Column': typemap['TExtArgumentType'].get('ColumnInt64'), + 'Column': typemap['TExtArgumentType'].get('ColumnFloat'), + 'Column': typemap['TExtArgumentType'].get('ColumnDouble'), + 'Column': typemap['TExtArgumentType'].get( + 'ColumnTextEncodingDict'), + 'Cursor': typemap['TExtArgumentType']['Cursor'], + 'void': typemap['TExtArgumentType']['Void'], + 'GeoPoint': typemap['TExtArgumentType'].get('GeoPoint'), + 'GeoLineString': typemap['TExtArgumentType'].get('GeoLineString'), + 'GeoPolygon': typemap['TExtArgumentType'].get('GeoPolygon'), + 'GeoMultiPolygon': typemap['TExtArgumentType'].get( + 'GeoMultiPolygon'), + 'Bytes': typemap['TExtArgumentType'].get('TextEncodingNone'), + 'TextEncodingDict': typemap['TExtArgumentType'].get('TextEncodingDict'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListBool'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt8'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt16'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt32'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt64'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListFloat'), + 'ColumnList': typemap['TExtArgumentType'].get('ColumnListDouble'), + 'ColumnList': typemap['TExtArgumentType'].get( + 'ColumnListTextEncodingDict'), + 'Timestamp': typemap['TExtArgumentType'].get('Timestamp'), + 'Column': typemap['TExtArgumentType'].get('ColumnTimestamp'), + } + + if self.version[:2] < (5, 4): + ext_arguments_map['Array'] = typemap[ + 'TExtArgumentType']['ArrayInt8'] + + ext_arguments_map['bool8'] = ext_arguments_map['bool'] + + for ptr_type, T in [ + ('bool', 'bool'), + ('bool8', 'bool'), + ('int8', 'int8_t'), + ('int16', 'int16_t'), + ('int32', 'int32_t'), + ('int64', 'int64_t'), + ('float32', 'float'), + ('float64', 'double'), + ('TextEncodingDict', 'TextEncodingDict'), + ('OmnisciTextEncodingDictType<>', 'TextEncodingDict'), + ('TimeStamp', 'TimeStamp'), + ]: + ext_arguments_map['OmnisciArrayType<%s>' % ptr_type] \ + = ext_arguments_map.get('Array<%s>' % T) + ext_arguments_map['OmnisciColumnType<%s>' % ptr_type] \ + = ext_arguments_map.get('Column<%s>' % T) + ext_arguments_map['OmnisciOutputColumnType<%s>' % ptr_type] \ + = ext_arguments_map.get('Column<%s>' % T) + ext_arguments_map['OmnisciColumnListType<%s>' % ptr_type] \ + = ext_arguments_map.get('ColumnList<%s>' % T) + ext_arguments_map['OmnisciOutputColumnListType<%s>' % ptr_type] \ + = ext_arguments_map.get('ColumnList<%s>' % T) + + ext_arguments_map['OmnisciBytesType'] = ext_arguments_map.get('Bytes') + + values = list(ext_arguments_map.values()) + for v, n in thrift.TExtArgumentType._VALUES_TO_NAMES.items(): + if v not in values: + warnings.warn('thrift.TExtArgumentType.%s(=%s) value not ' + 'in ext_arguments_map' % (n, v)) + self._ext_arguments_map = ext_arguments_map + return ext_arguments_map + + def type_to_extarg(self, t): + if isinstance(t, typesystem.Type): + s = t.tostring(use_annotation=False, use_name=False) + extarg = self._get_ext_arguments_map().get(s) + if extarg is None: + raise ValueError(f'cannot convert {t}(={s}) to ExtArgumentType') + return extarg + elif isinstance(t, str): + extarg = self._get_ext_arguments_map().get(t) + if extarg is None: + raise ValueError(f'cannot convert {t} to ExtArgumentType') + return extarg + elif isinstance(t, tuple): + return tuple(map(self.type_to_extarg, t)) + else: + raise TypeError(f'expected typesystem Type|tuple|str, got {type(t).__name__}') + + def retrieve_targets(self): + device_params = self.thrift_call('get_device_parameters', + self.session_id) + thrift = self.thrift_client.thrift + typemap = self.thrift_typemap + device_target_map = {} + messages = [] + for prop, value in device_params.items(): + if prop.endswith('_triple'): + device = prop.rsplit('_', 1)[0] + device_target_map[device] = value + + # Update thrift type map from server configuration + if 'Type.' in prop: + typ, member = prop.split('.', 1) + ivalue = int(value) + # Any warnings below indicate that the client and + # server thrift configurations are different. Server + # configuration will win. + if typ not in typemap: + messages.append(f'thrift type map: add new type {typ}') + elif member not in typemap[typ]: + messages.append( + f'thrift type map: add new member {typ}.{member}') + elif ivalue != typemap[typ][member]: + messages.append( + f'thrift type map: update {typ}.{member}' + f' to {ivalue} (was {typemap[typ][member]})') + else: + continue + typemap[typ][member] = ivalue + thrift_type = getattr(thrift, typ, None) + if thrift_type is not None: + thrift_type._VALUES_TO_NAMES[ivalue] = member + thrift_type._NAMES_TO_VALUES[member] = ivalue + self._ext_arguments_map = None + if messages: + warnings.warn('\n '.join([''] + messages)) + + type_sizeof = device_params.get('type_sizeof') + type_sizeof_dict = dict() + if type_sizeof is not None: + for type_size in type_sizeof.split(';'): + if not type_size: + continue + dtype, size = type_size.split(':') + type_sizeof_dict[dtype] = int(size) + + null_values = device_params.get('null_values') + null_values_asint = dict() + null_values_astype = dict() + if null_values is not None: + for tname_value in null_values.split(';'): + if not tname_value: + continue + tname, value = tname_value.split(':') + null_values_asint[tname] = int(value) + dtype = tname + if dtype.startswith('Array<'): + dtype = dtype[6:-1] + bitwidth = int(''.join(filter(str.isdigit, dtype))) + if bitwidth > 1: + null_value = numpy.dtype(f'uint{bitwidth}').type(int(value)).view( + 'int8' if dtype == 'boolean8' else dtype) + null_values_astype[tname] = null_value + self._null_values = null_values_astype + + targets = {} + for device, target in device_target_map.items(): + target_info = TargetInfo(name=device) + for prop, value in device_params.items(): + if not prop.startswith(device + '_'): + continue + target_info.set(prop[len(device)+1:], value) + if device == 'gpu': + self.has_cuda = True + # TODO: remove this hack + # see https://github.com/numba/numba/issues/4546 + target_info.set('name', 'skylake') + targets[device] = target_info + + if target_info.is_cpu: + target_info.set('driver', 'none') + target_info.add_library('m') + target_info.add_library('stdio') + target_info.add_library('stdlib') + target_info.add_library('omniscidb') + # NOTE: eventually, we want omniscidb to provide a + # 'free_buffer' function, but in the meantime we just call + # free() + target_info.set('fn_allocate_varlen_buffer', 'allocate_varlen_buffer') + target_info.set('fn_free_buffer', 'free') + elif target_info.is_gpu and self.version >= (5, 5): + target_info.add_library('libdevice') + + version_str = '.'.join(map(str, self.version[:3])) + self.version[3] + target_info.set('software', f'OmnisciDB {version_str}') + + llvm_version = device_params.get('llvm_version') + if llvm_version is not None: + target_info.set('llvm_version', tuple(map(int, llvm_version.split('.')))) + + if type_sizeof_dict is not None: + target_info.type_sizeof.update(type_sizeof_dict) + + target_info.set('null_values', null_values_asint) + return targets + + @property + def forbidden_names(self): + """Return a list of forbidden function names. See + https://github.com/xnd-project/rbc/issues/32 + """ + if self.version < (5, 2): + return ['sinh', 'cosh', 'tanh', 'rint', 'trunc', 'expm1', + 'exp2', 'log2', 'log1p', 'fmod'] + return [] + + def _make_udtf(self, caller, orig_sig, sig): + if self.version < (5, 4): + v = '.'.join(map(str, self.version)) + raise RuntimeError( + 'UDTFs with Column arguments require ' + 'omniscidb 5.4 or newer, currently ' + 'connected to ', v) + thrift = self.thrift_client.thrift + + inputArgTypes = [] + outputArgTypes = [] + sqlArgTypes = [] + annotations = [] + function_annotations = dict() # TODO: retrieve function annotations from orig_sig + sizer = None + sizer_index = -1 + + consumed_index = 0 + name = caller.func.__name__ + for i, a in enumerate(orig_sig[1]): + if is_sizer(a): + sizer = get_sizer_enum(a) + # cannot have multiple sizer arguments: + assert sizer_index == -1 + sizer_index = consumed_index + 1 + + annot = a.annotation() + + # process function annotations first to avoid appending annotations twice + if isinstance(a, OmnisciTableFunctionManagerType): + function_annotations['uses_manager'] = 'True' + consumed_index += 1 + continue + + annotations.append(annot) + + if isinstance(a, OmnisciCursorType): + sqlArgTypes.append(self.type_to_extarg('Cursor')) + for a_ in a.as_consumed_args: + assert not isinstance( + a_, OmnisciOutputColumnType), (a_) + inputArgTypes.append(self.type_to_extarg(a_)) + consumed_index += 1 + + else: + if isinstance(a, OmnisciOutputColumnType): + atype = self.type_to_extarg(a) + outputArgTypes.append(atype) + else: + atype = self.type_to_extarg(a) + if isinstance(a, (OmnisciColumnType, OmnisciColumnListType)): + sqlArgTypes.append(self.type_to_extarg('Cursor')) + inputArgTypes.append(atype) + else: + sqlArgTypes.append(atype) + inputArgTypes.append(atype) + consumed_index += 1 + if sizer is None: + sizer_index = get_literal_return(caller.func, verbose=self.debug) + if sizer_index is None: + sizer = 'kTableFunctionSpecifiedParameter' + else: + sizer = 'kConstant' + if sizer_index < 0: + raise ValueError( + f'Table function `{caller.func.__name__}`' + ' sizing parameter must be non-negative' + f' integer (got {sizer_index})') + sizer_type = (thrift.TOutputBufferSizeType + ._NAMES_TO_VALUES[sizer]) + annotations.append(function_annotations) + return thrift.TUserDefinedTableFunction( + name + sig.mangling(), + sizer_type, sizer_index, + inputArgTypes, outputArgTypes, sqlArgTypes, + annotations) + + def _make_udtf_old(self, caller, orig_sig, sig): + # old style UDTF for omniscidb <= 5.3, to be deprecated + if self.version >= (5, 4): + v = '.'.join(map(str, self.version)) + raise RuntimeError( + 'Old-style UDTFs require ' + 'omniscidb 5.3 or older, currently ' + 'connected to ', v) + thrift = self.thrift_client.thrift + + sizer = None + sizer_index = -1 + inputArgTypes = [] + outputArgTypes = [] + sqlArgTypes = [] + name = caller.func.__name__ + for i, a in enumerate(sig[1]): + _sizer = a.annotation().get('sizer') + if _sizer is not None: + # expect no more than one sizer argument + assert sizer_index == -1 + sizer_index = i + 1 + sizer = _sizer + atype = self.type_to_extarg(a) + if 'output' in a.annotation(): + outputArgTypes.append(atype) + else: + if 'input' in a.annotation(): + sqlArgTypes.append(atype) + elif 'cursor' in a.annotation(): + sqlArgTypes.append(self.type_to_extarg('Cursor')) + inputArgTypes.append(atype) + if sizer is None: + sizer = 'kConstant' + sizer_type = (thrift.TOutputBufferSizeType + ._NAMES_TO_VALUES[sizer]) + return thrift.TUserDefinedTableFunction( + name + sig.mangling(), + sizer_type, sizer_index, + inputArgTypes, outputArgTypes, sqlArgTypes) + + def _make_udf(self, caller, orig_sig, sig): + name = caller.func.__name__ + thrift = self.thrift_client.thrift + rtype = self.type_to_extarg(sig[0]) + atypes = self.type_to_extarg(sig[1]) + return thrift.TUserDefinedFunction( + name + sig.mangling(), + atypes, rtype) + + def register(self): + """Register caller cache to the server.""" + with typesystem.Type.alias(**self.typesystem_aliases): + return self._register() + + def _register(self): + if self.have_last_compile: + return + + device_ir_map = {} + llvm_function_names = [] + fid = 0 # UDF/UDTF id + udfs, udtfs = [], [] + for device, target_info in self.targets.items(): + with target_info: + udfs_map, udtfs_map = {}, {} + functions_and_signatures = [] + function_signatures = defaultdict(list) + for caller in reversed(self.get_callers()): + signatures = {} + name = caller.func.__name__ + if name in self.forbidden_names: + raise ForbiddenNameError( + f'\n\nAttempt to define function with name `{name}`.\n' + f'As a workaround, add a prefix to the function name ' + f'or define it with another name:\n\n' + f' def prefix_{name}(x):\n' + f' return np.trunc(x)\n\n' + f'For more information, see: ' + f'https://github.com/xnd-project/rbc/issues/32') + for sig in caller.get_signatures(): + i = len(function_signatures[name]) + if sig in function_signatures[name]: + if self.debug: + f2 = os.path.basename( + caller.func.__code__.co_filename) + n2 = caller.func.__code__.co_firstlineno + print(f'{type(self).__name__}.register: ignoring' + f' older definition of `{name}` for `{sig}`' + f' in {f2}#{n2}.') + continue + fid += 1 + orig_sig = sig + sig = sig[0](*sig.argument_types, **dict(name=name)) + function_signatures[name].append(sig) + sig_is_udtf = is_udtf(sig) + is_old_udtf = 'table' in sig[0].annotation() + + if i == 0 and (self.version < (5, 2) or is_old_udtf or + (self.version < (5, 5) and sig_is_udtf)): + sig.set_mangling('') + else: + if self.version < (5, 5): + sig.set_mangling('__%s' % (i)) + else: + sig.set_mangling('__%s_%s' % (device, i)) + + if sig_is_udtf: + # new style UDTF, requires omniscidb version >= 5.4 + udtfs_map[fid] = self._make_udtf(caller, orig_sig, sig) + elif is_old_udtf: + # old style UDTF for omniscidb <= 5.3, to be deprecated + udtfs_map[fid] = self._make_udtf_old(caller, orig_sig, sig) + else: + udfs_map[fid] = self._make_udf(caller, orig_sig, sig) + signatures[fid] = sig + functions_and_signatures.append((caller.func, signatures)) + + llvm_module, succesful_fids = compile_to_LLVM( + functions_and_signatures, + target_info, + pipeline_class=OmnisciCompilerPipeline, + user_defined_llvm_ir=self.user_defined_llvm_ir.get(device), + debug=self.debug) + + assert llvm_module.triple == target_info.triple + assert llvm_module.data_layout == target_info.datalayout + for f in llvm_module.functions: + llvm_function_names.append(f.name) + + device_ir_map[device] = str(llvm_module) + skipped_names = [] + for fid, udf in udfs_map.items(): + if fid in succesful_fids: + udfs.append(udf) + else: + skipped_names.append(udf.name) + + for fid, udtf in udtfs_map.items(): + if fid in succesful_fids: + udtfs.append(udtf) + else: + skipped_names.append(udtf.name) + if self.debug: + print(f'Skipping: {", ".join(skipped_names)}') + # Make sure that all registered functions have + # implementations, otherwise, we will crash the server. + for f in udtfs: + assert f.name in llvm_function_names + for f in udfs: + assert f.name in llvm_function_names + + if self.debug: + names = ", ".join([f.name for f in udfs] + [f.name for f in udtfs]) + print(f'Registering: {names}') + + self.set_last_compile(device_ir_map) + return self.thrift_call( + 'register_runtime_extension_functions', + self.session_id, udfs, udtfs, device_ir_map) + + def unregister(self): + """Unregister caller cache locally and on the server.""" + self.reset() + self.register() + + def preprocess_callable(self, func): + func = super().preprocess_callable(func) + if 'heavyai' not in func.__code__.co_names: + for symbol in BufferMeta.class_names: + if symbol in func.__code__.co_names and symbol not in func.__globals__: + warnings.warn( + f'{func.__name__} uses {symbol} that may be undefined.' + f' Inserting {symbol} to global namespace.' + f' Use `from rbc.heavyai import {symbol}`' + ' to remove this warning.') + func.__globals__[symbol] = heavyai.__dict__.get(symbol) + return func + + _compiler = None + + @property + def compiler(self): + """Return a C++/C to LLVM IR compiler instance. + """ + if self._compiler is None: + compiler = ctools.Compiler.get(std='c++14') + if compiler is None: # clang++ not available, try clang.. + compiler = ctools.Compiler.get(std='c') + if self.debug: + print(f'compiler={compiler}') + self._compiler = compiler + return self._compiler + + def caller_signature(self, signature: typesystem.Type): + """Return signature of a caller. + + See RemoteJIT.caller_signature.__doc__. + """ + if is_udtf(signature): + rtype = signature[0] + if not (rtype.is_int and rtype.bits == 32): + raise ValueError( + f'UDTF implementation return type must be int32, got {rtype}') + rtypes = [] + atypes = [] + for atype in signature[1]: + if is_sizer(atype): + sizer = get_sizer_enum(atype) + if sizer not in user_specified_output_buffer_sizers: + continue + atype.annotation(sizer=sizer) + elif isinstance(atype, OmnisciTableFunctionManagerType): + continue + elif isinstance(atype, OmnisciOutputColumnType): + rtypes.append(atype.copy(OmnisciColumnType)) + continue + atypes.append(atype) + rtype = typesystem.Type(*rtypes, **dict(struct_is_tuple=True)) + return rtype(*atypes, **signature._params) + return signature + + def get_types(self, *values): + """Convert values to the corresponding typesystem types. + + See RemoteJIT.get_types.__doc__. + """ + types = [] + for value in values: + if isinstance(value, RemoteCallCapsule): + typ = value.__typesystem_type__ + if typ.is_struct and typ._params.get('struct_is_tuple'): + types.extend(typ) + else: + types.append(typ) + else: + types.append(typesystem.Type.fromvalue(value)) + return tuple(types) + + def normalize_function_type(self, ftype: typesystem.Type): + """Normalize typesystem function type. + + See RemoteJIT.normalize_function_type.__doc__. + """ + assert ftype.is_function, ftype + for atype in ftype[1]: + # convert `T foo` to `T | name=foo` + if 'name' not in atype.annotation() and atype.name is not None: + atype.annotation(name=atype.name) + atype._params.pop('name') + return ftype + + def format_type(self, typ: typesystem.Type): + """Convert typesystem type to formatted string. + + See RemoteJIT.format_type.__doc__. + """ + if typ.is_function: + args = map(self.format_type, typ[1]) + if is_udtf(typ): + return f'UDTF({", ".join(args)})' + else: + return f'({", ".join(args)}) -> {self.format_type(typ[0])}' + use_typename = False + if typ.is_struct and typ._params.get('struct_is_tuple'): + return f'({", ".join(map(self.format_type, typ))})' + if isinstance(typ, OmnisciOutputColumnType): + p = tuple(map(self.format_type, typ[0])) + typ = typesystem.Type(('OutputColumn',) + p, **typ._params) + elif isinstance(typ, OmnisciColumnType): + p = tuple(map(self.format_type, typ[0])) + typ = typesystem.Type(('Column',) + p, **typ._params) + elif isinstance(typ, OmnisciColumnListType): + p = tuple(map(self.format_type, typ[0])) + typ = typesystem.Type(('ColumnList',) + p, **typ._params) + elif isinstance(typ, OmnisciArrayType): + p = tuple(map(self.format_type, typ[0])) + typ = typesystem.Type(('Array',) + p, **typ._params) + elif isinstance(typ, OmnisciCursorType): + p = tuple(map(self.format_type, typ[0])) + typ = typesystem.Type(('Cursor',) + p, **typ._params) + elif isinstance(typ, OmnisciBytesType): + typ = typ.copy().params(typename='Bytes') + use_typename = True + elif isinstance(typ, OmnisciTextEncodingDictType): + typ = typ.copy().params(typename='TextEncodingDict') + use_typename = True + elif isinstance(typ, OmnisciTableFunctionManagerType): + typ = typ.copy().params(typename='TableFunctionManager') + use_typename = True + elif is_sizer(typ): + sizer = get_sizer_enum(typ) + for shortname, fullname in output_buffer_sizer_map.items(): + if fullname == sizer: + use_typename = True + typ = typ.copy().params(typename=shortname) + typ.annotation().pop('sizer') + break + + return typ.tostring(use_typename=use_typename, use_annotation_name=True) + + # We define remote_compile and remote_call for Caller.__call__ method. + def remote_compile(self, func, ftype: typesystem.Type, target_info: TargetInfo): + """Remote compile function and signatures to machine code. + + See RemoteJIT.remote_compile.__doc__. + """ + if self.query_requires_register(func.__name__): + self.register() + + def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False): + """ + See RemoteJIT.remote_call.__doc__. + """ + sig = self.caller_signature(ftype) + assert len(arguments) == len(sig[1]), (sig, arguments) + rtype = sig[0] + args = [] + for a, atype in zip(arguments, sig[1]): + if isinstance(a, RemoteCallCapsule): + if is_udtf(a.ftype): + a = a.execute(hold=True) + else: + a = a.execute(hold=True).lstrip('SELECT ') + + if isinstance(atype, (OmnisciColumnType, OmnisciColumnListType)): + args.append(f'CURSOR({a})') + elif isinstance(atype, OmnisciBytesType): + if isinstance(a, bytes): + a = repr(a.decode()) + elif isinstance(a, str): + a = repr(a) + args.append(f'{a}') + else: + args.append(f'CAST({a} AS {type_to_type_name(atype)})') + args = ', '.join(args) + is_udtf_call = is_udtf(ftype) + if is_udtf_call: + colnames = [] + if rtype.is_struct and rtype._params.get('struct_is_tuple'): + for i, t in enumerate(rtype): + n = t.annotation().get('name', f'out{i}') + colnames.append(n) + else: + colnames.append(rtype.annotation().get('name', '*')) + q = f'SELECT {", ".join(colnames)} FROM TABLE({func.__name__}({args}))' + else: + q = f'SELECT {func.__name__}({args})' + if hold: + return q + descrs, result = self.sql_execute(q + ';') + dtype = [(descr.name, type_name_to_dtype(descr.type_name)) for descr in descrs] + if is_udtf_call: + return numpy.array(list(result), dtype).view(numpy.recarray) + else: + return dtype[0][1](list(result)[0][0]) + + +class RemoteOmnisci(RemoteHeavyDB): + """Omnisci - the previous brand of HeavyAI + """ diff --git a/rbc/omnisci_backend/__init__.py b/rbc/omnisci_backend/__init__.py deleted file mode 100644 index afc8e168..00000000 --- a/rbc/omnisci_backend/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from .omnisci_array import * # noqa: F401, F403 -from .omnisci_column import * # noqa: F401, F403 -from .omnisci_bytes import * # noqa: F401, F403 -from .omnisci_metatype import * # noqa: F401, F403 -from .omnisci_text_encoding import * # noqa: F401, F403 -from .omnisci_pipeline import * # noqa: F401, F403 -from .omnisci_column_list import * # noqa: F401, F403 -from .omnisci_table_function_manager import * # noqa: F401, F403 - -import rbc.omnisci_backend.mathimpl as math # noqa: F401 -import rbc.omnisci_backend.npyimpl as np # noqa: F401 -import rbc.omnisci_backend.python_operators as operators # noqa: F401 - -# initialize the array api -from rbc.stdlib import array_api # noqa: F401 - -__all__ = [s for s in dir() if not s.startswith('_')] diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index 681c4e30..b849a218 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -1,1479 +1,6 @@ -"""OmniSciDB client config functions -""" +import warnings # noqa: F401 +from .heavydb import * # noqa: F401, F403 -import ast -import inspect -import os -import re -import warnings -import configparser -import numpy -from collections import defaultdict, namedtuple -from .remotejit import RemoteJIT, RemoteCallCapsule -from .thrift.utils import resolve_includes -from .thrift import Client as ThriftClient -from . import omnisci_backend -from .omnisci_backend import ( - OmnisciArrayType, OmnisciBytesType, OmnisciTextEncodingDictType, - OmnisciOutputColumnType, OmnisciColumnType, - OmnisciCompilerPipeline, OmnisciCursorType, - BufferMeta, OmnisciColumnListType, OmnisciTableFunctionManagerType) -from .targetinfo import TargetInfo -from .irtools import compile_to_LLVM -from .errors import ForbiddenNameError, OmnisciServerError -from .utils import parse_version, version_date -from . import ctools -from . import typesystem - -def get_literal_return(func, verbose=False): - """Return a literal value from the last function return statement or - None. - """ - def _convert(node): - if isinstance(node, (int, float, complex, str, tuple, list, dict)): - return node - if isinstance(node, ast.Constant): - return node.value - if isinstance(node, ast.Num): - return _convert(node.n) - if isinstance(node, ast.Name): - if node.id in func.__code__.co_names: - return func.__globals__[node.id] - args = func.__code__.co_varnames[:func.__code__.co_argcount] - args_with_default = args[len(args) - len(func.__defaults__):] - if node.id in args_with_default: - return func.__defaults__[ - list(args_with_default).index(node.id)] - raise NotImplementedError(f'literal value from name `{node.id}`') - if isinstance(node, ast.BinOp): - if isinstance(node.op, ast.Add): - return _convert(node.left) + _convert(node.right) - if isinstance(node.op, ast.Sub): - return _convert(node.left) - _convert(node.right) - if isinstance(node.op, ast.Mult): - return _convert(node.left) * _convert(node.right) - if isinstance(node.op, ast.Div): - return _convert(node.left) / _convert(node.right) - if isinstance(node.op, ast.FloorDiv): - return _convert(node.left) // _convert(node.right) - if isinstance(node.op, ast.Mod): - return _convert(node.left) % _convert(node.right) - if isinstance(node.op, ast.Pow): - return _convert(node.left) ** _convert(node.right) - raise NotImplementedError( - f'literal value from binary op `{node.op}`') - raise NotImplementedError(f'literal value from `{node}`') - source = '' - intent = None - for line in inspect.getsourcelines(func)[0]: - if line.lstrip().startswith('@'): - continue - if intent is None: - intent = len(line) - len(line.lstrip()) - source += line[intent:] - - tree = ast.parse(source) - last_node = tree.body[0].body[-1] - if isinstance(last_node, ast.Return): - try: - return _convert(last_node.value) - except Exception as msg: - if verbose: - print(f'get_literal_return: {msg}') - print(source) - - -def _global_omnisci(): - """Implements singleton of a global RemoteOmnisci instance. - """ - config = get_client_config() - remotedb = dict(heavyai=RemoteHeavyDB, - omnisci=RemoteOmnisci)[config['dbname']](**config) - while True: - yield remotedb - - -global_omnisci_singleton = _global_omnisci() # generator object - - -def is_available(_cache={}): - """Return version tuple and None if HeavyDB/OmnisciDB server is - accessible or recent enough. Otherwise return None and the reason - about unavailability. - """ - - if not _cache: - remotedb = next(global_omnisci_singleton) - try: - version = remotedb.version - except Exception as msg: - _cache['reason'] = f'failed to get {type(remotedb).__name__} version: {msg}' - else: - print(f' {type(remotedb).__name__} version {version}') - if version[:2] >= (4, 6): - _cache['version'] = version - else: - _cache['reason'] = ( - f'expected {type(remotedb).__name__} version 4.6 or greater, got {version}') - return _cache.get('version', ()), _cache.get('reason', '') - - -def get_client_config(**config): - """Retrieve the HeavyDB client configuration parameters from a client - home directory. - - Two HeavyDB brands (HEAVYDB_BRAND) are supported: heavyai and - omnisci. - - Note that here the client configurations parameters are those that - are used to configure the client software such as rbc or pymapd. - This is different from heavydb instance configuration described in - https://docs.heavy.ai/installation-and-configuration/config-parameters - that is used for configuring the heavydb server software. - - In Linux clients, the HeavyDB client configuration is read from - :code:`$HOME/.config/$HEAVYDB_BRAND/client.conf` - - In Windows clients, the configuration is read from - :code:`%UserProfile/.config/%HEAVYDB_BRAND%/client.conf` or - :code:`%AllUsersProfile/.config/%HEAVYDB_BRAND%/client.conf` - - When :code:`HEAVYDB_CLIENT_CONF` or :code:`OMNISCI_CLIENT_CONF` - environment variable is defined then the configuration is read - from the file specified in this variable. - - The configuration file must use configuration language similar to - one used in MS Windows INI files. For HeavyDB client - configuration, the file may contain, for instance:: - - [user] - name: - password: - - [server] - host: - port: - dbname: - - Parameters - ---------- - config : dict - Specify configuration parameters that override the parameters - from configuration file. - - Returns - ------- - config : dict - A dictionary of `user`, `password`, `host`, `port`, `dbname` and - other RemoteJIT options. - """ - caller_config = config - _config = dict(user='admin', password='HyperInteractive', - host='localhost', port=6274) - _config.update(**config) - config = _config - - conf_file = None - for brand, client_conf_env in [('heavyai', 'HEAVYDB_CLIENT_CONF'), - ('omnisci', 'OMNISCI_CLIENT_CONF')]: - conf_file = os.environ.get(client_conf_env, None) - if conf_file is not None and not os.path.isfile(conf_file): - print('rbc.omnisci.get_client_config:' - f' {client_conf_env}={conf_file!r}' - ' is not a file, ignoring.') - conf_file = None - if conf_file is None: - conf_file_base = os.path.join('.config', brand, 'client.conf') - for prefix_env in ['UserProfile', 'AllUsersProfile', 'HOME']: - prefix = os.environ.get(prefix_env, None) - if prefix is not None: - fn = os.path.join(prefix, conf_file_base) - if os.path.isfile(fn): - conf_file = fn - break - if conf_file is not None: - break - - if conf_file is not None: - conf = configparser.ConfigParser() - conf.read(conf_file) - - if 'user' in conf: - user = conf['user'] - if 'name' in user and 'name' not in caller_config: - config['user'] = user['name'] - if 'password' in user and 'password' not in caller_config: - config['password'] = user['password'] - - if 'server' in conf: - server = conf['server'] - if 'host' in server and 'host' not in caller_config: - config['host'] = server['host'] - if 'port' in server and 'port' not in caller_config: - config['port'] = int(server['port']) - if 'dbname' in server and 'dbname' not in caller_config: - config['dbname'] = server['dbname'] - - if 'rbc' in conf: - rbc = conf['rbc'] - for k in ['debug', 'use_host_target']: - if k in rbc and k not in caller_config: - config[k] = rbc.getboolean(k) - - if 'dbname' not in config: - version = get_heavydb_version(host=config['host'], port=config['port']) - if version is not None and version[:2] >= (6, 0): - if version[:3] == (6, 0, 0) and version_date(version) < 20220301: - # TODO: remove this if-block when heavydb 6.0 is released. - config['dbname'] = 'omnisci' - else: - config['dbname'] = 'heavyai' - else: - config['dbname'] = 'omnisci' - - return config - - -def is_udtf(sig): - """Check if signature is a table function signature. - """ - if sig[0].annotation().get('kind') == 'UDTF': - return True - for a in sig[1]: - if isinstance(a, (OmnisciOutputColumnType, OmnisciColumnType, - OmnisciColumnListType, OmnisciTableFunctionManagerType)): - return True - return False - - -def is_sizer(t): - """Check if type is a type of a sizer argument: - int32_t | sizer=... - """ - return t.is_int and t.bits == 32 and 'sizer' in t.annotation() - - -def get_sizer_enum(t): - """Return sizer enum value as defined by the omniscidb server. - """ - sizer = t.annotation()['sizer'] - sizer = output_buffer_sizer_map.get(sizer or None, sizer) - for shortname, fullname in output_buffer_sizer_map.items(): - if sizer == fullname: - return sizer - raise ValueError(f'unknown sizer value ({sizer}) in {t}') - - -output_buffer_sizer_map = dict( - ConstantParameter='kUserSpecifiedConstantParameter', - RowMultiplier='kUserSpecifiedRowMultiplier', - Constant='kConstant', - SpecifiedParameter='kTableFunctionSpecifiedParameter', - PreFlight='kPreFlightParameter') - -# Default sizer is RowMultiplier: -output_buffer_sizer_map[None] = output_buffer_sizer_map['RowMultiplier'] - -user_specified_output_buffer_sizers = { - 'kUserSpecifiedConstantParameter', 'kUserSpecifiedRowMultiplier', -} - - -def type_to_type_name(typ: typesystem.Type): - """Return typesystem.Type as DatumType name. - """ - styp = typ.tostring(use_annotation=False, use_name=False) - type_name = dict( - int8='TINYINT', - int16='SMALLINT', - int32='INT', - int64='BIGINT', - float32='FLOAT', - float64='DOUBLE', - ).get(styp) - if type_name is not None: - return type_name - raise NotImplementedError(f'converting `{styp}` to DatumType not supported') - - -def type_name_to_dtype(type_name): - """Return DatumType name as the corresponding numpy dtype. - """ - dtype = dict( - SMALLINT=numpy.int16, - INT=numpy.int32, - BIGINT=numpy.int64, - FLOAT=numpy.float32, - # DECIMAL=, - DOUBLE=numpy.float32, - STR=numpy.str0, - # TIME=, - # TIMESTAMP=, - # DATE=, - BOOL=numpy.bool8, - # INTERVAL_DAY_TIME=, - # INTERVAL_YEAR_MONTH=, - # POINT=, - # LINESTRING=, - # POLYGON=, - # MULTIPOLYGON=, - TINYINT=numpy.int8, - # GEOMETRY=, - # GEOGRAPHY= - ).get(type_name) - if dtype is not None: - return dtype - raise NotImplementedError( - f'convert DatumType `{type_name}` to numpy dtype') - - -class OmnisciQueryCapsule(RemoteCallCapsule): - - use_execute_cache = True - - def __repr__(self): - return f'{type(self).__name__}({str(self)!r})' - - -def get_heavydb_version(host='localhost', port=6274, _cache={}): - """Acquires the version of heavydb server. - """ - if (host, port) in _cache: - return _cache[host, port] - thrift_content = ''' -exception TMapDException { - 1: string error_msg -} -service Omnisci { - string get_version() throws (1: TMapDException e) -} -''' - client = ThriftClient( - host=host, - port=port, - multiplexed=False, - thrift_content=thrift_content, - socket_timeout=60000) - try: - version = client(Omnisci=dict(get_version=()))['Omnisci']['get_version'] - except Exception as msg: - print(f'failed to get heavydb version[host={host}, port={port}]: {msg}') - version = None - else: - version = parse_version(version) - _cache[host, port] = version - return version - - -class RemoteHeavyDB(RemoteJIT): - - """Usage: - - .. code:: python - - omnisci = RemoteOmnisci(host=..., port=...) - - @omnisci('int(int, int)') - def add(a, b): - return a + b - - omnisci.register() - - Use pymapd, for instance, to make a SQL query `select add(c1, - c2) from table` - """ - multiplexed = False - mangle_prefix = '' - - typesystem_aliases = dict( - bool='bool8', - Array='OmnisciArrayType', - Bytes='OmnisciBytesType', - Cursor='OmnisciCursorType', - Column='OmnisciColumnType', - OutputColumn='OmnisciOutputColumnType', - RowMultiplier='int32|sizer=RowMultiplier', - ConstantParameter='int32|sizer=ConstantParameter', - SpecifiedParameter='int32|sizer=SpecifiedParameter', - Constant='int32|sizer=Constant', - PreFlight='int32|sizer=PreFlight', - ColumnList='OmnisciColumnListType', - TextEncodingDict='OmnisciTextEncodingDictType', - TableFunctionManager='OmnisciTableFunctionManagerType<>', - UDTF='int32|kind=UDTF' - ) - - remote_call_capsule_cls = OmnisciQueryCapsule - default_remote_call_hold = True - supports_local_caller = False - - def __init__(self, - user='admin', - password='HyperInteractive', - host='localhost', - port=6274, - dbname='omnisci', - **options): - self.user = user - self.password = password - self.dbname = dbname - - thrift_filename = os.path.join(os.path.dirname(__file__), - 'omnisci.thrift') - content = resolve_includes( - open(thrift_filename).read(), - [os.path.dirname(thrift_filename)]) - for p in ['common.', 'extension_functions.']: - content = content.replace(p, '') - self.thrift_content = content - - RemoteJIT.__init__(self, host=host, port=port, **options) - - self._version = None - self._thrift_client = None - self._session_id = None - self._targets = None - self.thrift_typemap = defaultdict(dict) - self._init_thrift_typemap() - self.has_cuda = None - self._null_values = dict() - - # An user-defined device-LLVM IR mapping. - self.user_defined_llvm_ir = {} - - def _init_thrift_typemap(self): - """Initialize thrift type map using client thrift configuration. - """ - typemap = self.thrift_typemap - for typename, typ in self.thrift_client.thrift.__dict__.items(): - if hasattr(typ, '_NAMES_TO_VALUES'): - for name, value in typ._NAMES_TO_VALUES.items(): - typemap[typename][name] = value - typemap[typename+'-inverse'][value] = name - - @property - def version(self): - if self._version is None: - version = self.thrift_call('get_version') - self._version = parse_version(version) - if self._version[:2] < (5, 6): - msg = (f'OmniSciDB server v.{version} is too old (expected v.5.6 or newer) ' - 'and some features might not be available.') - warnings.warn(msg, PendingDeprecationWarning) - return self._version - - @property - def session_id(self): - if self._session_id is None: - user = self.user - pw = self.password - dbname = self.dbname - self._session_id = self.thrift_call('connect', user, pw, dbname) - return self._session_id - - @property - def thrift_client(self): - if self._thrift_client is None: - self._thrift_client = self.client - return self._thrift_client - - def thrift_call(self, name, *args, **kwargs): - client = kwargs.get('client') - if client is None: - client = self.thrift_client - - if name == 'register_runtime_udf' and self.version[:2] >= (4, 9): - warnings.warn('Using register_runtime_udf is deprecated, ' - 'use register_runtime_extension_functions instead.') - session_id, signatures, device_ir_map = args - udfs = [] - sig_re = re.compile(r"\A(?P[\w\d_]+)\s+" - r"'(?P[\w\d_]+)[(](?P.*)[)]'\Z") - thrift = client.thrift - ext_arguments_map = self._get_ext_arguments_map() - for signature in signatures.splitlines(): - m = sig_re.match(signature) - if m is None: - raise RuntimeError( - 'Failed to parse signature %r' % (signature)) - name = m.group('name') - rtype = ext_arguments_map[m.group('rtype')] - atypes = [ext_arguments_map[a.strip()] - for a in m.group('atypes').split(',')] - udfs.append(thrift.TUserDefinedFunction( - name, atypes, rtype)) - return self.thrift_call('register_runtime_extension_functions', - session_id, udfs, [], device_ir_map) - - if self.debug: - msg = 'thrift_call %s%s' % (name, args) - if len(msg) > 200: - msg = msg[:180] + '...' + msg[-15:] - print(msg) - try: - return client(Omnisci={name: args})['Omnisci'][name] - except client.thrift.TMapDException as msg: - m = re.match(r'.*CalciteContextException.*(' - r'No match found for function signature.*' - r'|Cannot apply.*)', - msg.error_msg) - if m: - raise OmnisciServerError(f'[Calcite] {m.group(1)}') - m = re.match(r'.*Exception: (.*)', msg.error_msg) - if m: - raise OmnisciServerError(f'{m.group(1)}') - m = re.match( - r'(.*)\: No match found for function signature (.*)\(.*\)', - msg.error_msg - ) - if m: - msg = (f"Undefined function call {m.group(2)!r} in" - f" SQL statement: {m.group(1)}") - raise OmnisciServerError(msg) - m = re.match(r'.*SQL Error: (.*)', msg.error_msg) - if m: - raise OmnisciServerError(f'{m.group(1)}') - m = re.match(r'Could not bind *', msg.error_msg) - if m: - raise OmnisciServerError(msg.error_msg) - m = re.match(r'Runtime extension functions registration is disabled.', - msg.error_msg) - if m: - msg = (f"{msg.error_msg} Please use server options --enable-runtime-udf" - " and/or --enable-table-functions") - raise OmnisciServerError(msg) - - # TODO: catch more known server failures here. - raise - - def get_tables(self): - """Return a list of table names stored in the OmnisciDB server. - """ - return self.thrift_call('get_tables', self.session_id) - - def get_table_details(self, table_name): - """Return details about OmnisciDB table. - """ - return self.thrift_call('get_table_details', - self.session_id, table_name) - - def load_table_columnar(self, table_name, **columnar_data): - """Load columnar data to OmnisciDB table. - - Warning: when connected to OmnisciDB < 5.3, the data is loaded - row-wise that will be very slow for large data sets. - - Parameters - ---------- - table_name : str - The name of table into the data is loaded. The table must exist. - - columnar_data : dict - A dictionary of column names and the corresponding column - values. A column values is a sequence of values. - The table must have the specified columns defined. - - """ - if not self._null_values: - self.retrieve_targets() # initializes null values - - int_col_types = ['TINYINT', 'SMALLINT', 'INT', 'BIGINT', 'BOOL', - 'DECIMAL', 'TIME', 'TIMESTAMP', 'DATE'] - real_col_types = ['FLOAT', 'DOUBLE'] - str_col_types = ['STR', 'POINT', 'LINESTRING', 'POLYGON', - 'MULTIPOLYGON'] - - datumtype_map = { - 'BOOL': 'boolean8', - 'TINYINT': 'int8', - 'SMALLINT': 'int16', - 'INT': 'int32', - 'BIGINT': 'int64', - 'FLOAT': 'float32', - 'DOUBLE': 'float64', - 'BOOL[]': 'Array', - 'TINYINT[]': 'Array', - 'SMALLINT[]': 'Array', - 'INT[]': 'Array', - 'BIGINT[]': 'Array', - 'FLOAT[]': 'Array', - 'DOUBLE[]': 'Array', - } - - thrift = self.thrift_client.thrift - table_details = self.get_table_details(table_name) - use_sql = self.version[:2] < (5, 3) - if not use_sql and self.version[:2] < (5, 5): - for column_data in columnar_data.values(): - for v in column_data: - if v is None: - break - elif isinstance(v, (tuple, list)): - for _v in v: - if _v is None: - break - else: - continue - break - else: - continue - use_sql = True - break - if use_sql: - rows = None - for column_name, column_data in columnar_data.items(): - if rows is None: - rows = [[None for j in range(len(columnar_data))] - for i in range(len(column_data))] - col_index = None - for i, ct in enumerate(table_details.row_desc): - if ct.col_name == column_name: - typeinfo = ct.col_type - col_index = i - datumtype = thrift.TDatumType._VALUES_TO_NAMES[ - typeinfo.type] - is_array = typeinfo.is_array - break - assert col_index is not None - for i, v in enumerate(column_data): - if v is None: - v = "NULL" - elif is_array: - if datumtype == 'BOOL': - v = ["'true'" if v_ else ("'false'" - if v_ is not None else "NULL") - for v_ in v] - else: - v = [(str(v_) if v_ is not None else "NULL") for v_ in v] - v = ', '.join(v) - v = f'ARRAY[{v}]' - else: - if datumtype == 'BOOL': - v = "'true'" if v else "'false'" - rows[i][col_index] = v - - for row in rows: - table_row = ', '.join(map(str, row)) - self.sql_execute( - f'INSERT INTO {table_name} VALUES ({table_row})') - return - - columns = [] - for column_name, column_data in columnar_data.items(): - typeinfo = None - for ct in table_details.row_desc: - if ct.col_name == column_name: - typeinfo = ct.col_type - break - if typeinfo is None: - raise ValueError( - f'OmnisciDB `{table_name}` has no column `{column_name}`') - datumtype = thrift.TDatumType._VALUES_TO_NAMES[typeinfo.type] - nulls = [v is None for v in column_data] - if typeinfo.is_array: - arrays = [] - for arr in column_data: - if arr is None: - arr_nulls = None - else: - arr_nulls = [v is None for v in arr] - if True in arr_nulls: - # TODO: null support for text - null_value = self._null_values[datumtype_map[datumtype]] - arr = [(null_value if v is None else v) for v in arr] - if datumtype in int_col_types: - arr_data = thrift.TColumnData(int_col=arr) - elif datumtype in real_col_types: - arr_data = thrift.TColumnData(real_col=arr) - elif datumtype in str_col_types: - arr_data = thrift.TColumnData(str_col=arr) - else: - raise NotImplementedError( - f'loading {datumtype} array data') - arrays.append(thrift.TColumn(data=arr_data, nulls=arr_nulls)) - col_data = thrift.TColumnData(arr_col=arrays) - else: - if True in nulls: - # TODO: null support for text - null_value = self._null_values[datumtype_map[datumtype]] - column_data = [(null_value if v is None else v) for v in column_data] - if datumtype in int_col_types: - col_data = thrift.TColumnData(int_col=column_data) - elif datumtype in real_col_types: - col_data = thrift.TColumnData(real_col=column_data) - elif datumtype in str_col_types: - col_data = thrift.TColumnData(str_col=column_data) - else: - raise NotImplementedError(f'loading {datumtype} data') - columns.append(thrift.TColumn(data=col_data, nulls=nulls)) - - self.thrift_call('load_table_binary_columnar', - self.session_id, table_name, columns) - - def _make_row_results_set(self, data): - # The following code is a stripped copy from omnisci/pymapd - - _typeattr = { - 'SMALLINT': 'int', - 'INT': 'int', - 'BIGINT': 'int', - 'TIME': 'int', - 'TIMESTAMP': 'int', - 'DATE': 'int', - 'BOOL': 'int', - 'FLOAT': 'real', - 'DECIMAL': 'real', - 'DOUBLE': 'real', - 'STR': 'str', - 'POINT': 'str', - 'LINESTRING': 'str', - 'POLYGON': 'str', - 'MULTIPOLYGON': 'str', - 'TINYINT': 'int', - 'GEOMETRY': 'str', - 'GEOGRAPHY': 'str', - } - - thrift = self.thrift_client.thrift - - def extract_col_vals(desc, val): - typename = thrift.TDatumType._VALUES_TO_NAMES[desc.col_type.type] - nulls = val.nulls - if hasattr(val.data, 'arr_col') and val.data.arr_col: - vals = [ - None if null else [ - None if _n else _v - for _n, _v in zip( - v.nulls, getattr(v.data, _typeattr[typename] + '_col'))] - for null, v in zip(nulls, val.data.arr_col) - ] - else: - vals = getattr(val.data, _typeattr[typename] + '_col') - vals = [None if null else v for null, v in zip(nulls, vals)] - return vals - - if data.row_set.columns: - nrows = len(data.row_set.columns[0].nulls) - ncols = len(data.row_set.row_desc) - columns = [ - extract_col_vals(desc, col) - for desc, col in zip(data.row_set.row_desc, data.row_set.columns) - ] - for i in range(nrows): - yield tuple(columns[j][i] for j in range(ncols)) - - def query_requires_register(self, query): - """Check if given query requires registration step. - """ - names = '|'.join(self.get_pending_names()) - return names and re.search(r'\b(' + names + r')\b', query, re.I) is not None - - def sql_execute(self, query): - """Execute SQL query in OmnisciDB server. - - Parameters - ---------- - query : str - SQL query string containing exactly one query. Multiple - queries are not supported. - - Returns - ------- - descr : object - Row description object - results : iterator - Iterator over rows. - - """ - if self.query_requires_register(query): - self.register() - columnar = True - if self.debug: - print(' %s;' % (query)) - result = self.thrift_call( - 'sql_execute', self.session_id, query, columnar, "", -1, -1) - - type_code_to_type_name = self.thrift_typemap['TDatumType-inverse'] - Description = namedtuple("Description", ["name", "type_name", "null_ok"]) - descr = [] - for col in result.row_set.row_desc: - descr.append(Description(col.col_name, - type_code_to_type_name[col.col_type.type], - col.col_type.nullable)) - return descr, self._make_row_results_set(result) - - _ext_arguments_map = None - - def _get_ext_arguments_map(self): - if self._ext_arguments_map is not None: - return self._ext_arguments_map - thrift = self.thrift_client.thrift - typemap = self.thrift_typemap - ext_arguments_map = { - 'int8': typemap['TExtArgumentType']['Int8'], - 'int16': typemap['TExtArgumentType']['Int16'], - 'int32': typemap['TExtArgumentType']['Int32'], - 'int64': typemap['TExtArgumentType']['Int64'], - 'float32': typemap['TExtArgumentType']['Float'], - 'float64': typemap['TExtArgumentType']['Double'], - 'int8*': typemap['TExtArgumentType']['PInt8'], - 'int16*': typemap['TExtArgumentType']['PInt16'], - 'int32*': typemap['TExtArgumentType']['PInt32'], - 'int64*': typemap['TExtArgumentType']['PInt64'], - 'float32*': typemap['TExtArgumentType']['PFloat'], - 'float64*': typemap['TExtArgumentType']['PDouble'], - 'bool': typemap['TExtArgumentType']['Bool'], - 'bool*': typemap['TExtArgumentType'].get('PBool'), - 'Array': typemap['TExtArgumentType'].get( - 'ArrayBool', typemap['TExtArgumentType']['ArrayInt8']), - 'Array': typemap['TExtArgumentType']['ArrayInt8'], - 'Array': typemap['TExtArgumentType']['ArrayInt16'], - 'Array': typemap['TExtArgumentType']['ArrayInt32'], - 'Array': typemap['TExtArgumentType']['ArrayInt64'], - 'Array': typemap['TExtArgumentType']['ArrayFloat'], - 'Array': typemap['TExtArgumentType']['ArrayDouble'], - 'Column': typemap['TExtArgumentType'].get('ColumnBool'), - 'Column': typemap['TExtArgumentType'].get('ColumnInt8'), - 'Column': typemap['TExtArgumentType'].get('ColumnInt16'), - 'Column': typemap['TExtArgumentType'].get('ColumnInt32'), - 'Column': typemap['TExtArgumentType'].get('ColumnInt64'), - 'Column': typemap['TExtArgumentType'].get('ColumnFloat'), - 'Column': typemap['TExtArgumentType'].get('ColumnDouble'), - 'Column': typemap['TExtArgumentType'].get( - 'ColumnTextEncodingDict'), - 'Cursor': typemap['TExtArgumentType']['Cursor'], - 'void': typemap['TExtArgumentType']['Void'], - 'GeoPoint': typemap['TExtArgumentType'].get('GeoPoint'), - 'GeoLineString': typemap['TExtArgumentType'].get('GeoLineString'), - 'GeoPolygon': typemap['TExtArgumentType'].get('GeoPolygon'), - 'GeoMultiPolygon': typemap['TExtArgumentType'].get( - 'GeoMultiPolygon'), - 'Bytes': typemap['TExtArgumentType'].get('TextEncodingNone'), - 'TextEncodingDict': typemap['TExtArgumentType'].get('TextEncodingDict'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListBool'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt8'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt16'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt32'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListInt64'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListFloat'), - 'ColumnList': typemap['TExtArgumentType'].get('ColumnListDouble'), - 'ColumnList': typemap['TExtArgumentType'].get( - 'ColumnListTextEncodingDict'), - 'Timestamp': typemap['TExtArgumentType'].get('Timestamp'), - 'Column': typemap['TExtArgumentType'].get('ColumnTimestamp'), - } - - if self.version[:2] < (5, 4): - ext_arguments_map['Array'] = typemap[ - 'TExtArgumentType']['ArrayInt8'] - - ext_arguments_map['bool8'] = ext_arguments_map['bool'] - - for ptr_type, T in [ - ('bool', 'bool'), - ('bool8', 'bool'), - ('int8', 'int8_t'), - ('int16', 'int16_t'), - ('int32', 'int32_t'), - ('int64', 'int64_t'), - ('float32', 'float'), - ('float64', 'double'), - ('TextEncodingDict', 'TextEncodingDict'), - ('OmnisciTextEncodingDictType<>', 'TextEncodingDict'), - ('TimeStamp', 'TimeStamp'), - ]: - ext_arguments_map['OmnisciArrayType<%s>' % ptr_type] \ - = ext_arguments_map.get('Array<%s>' % T) - ext_arguments_map['OmnisciColumnType<%s>' % ptr_type] \ - = ext_arguments_map.get('Column<%s>' % T) - ext_arguments_map['OmnisciOutputColumnType<%s>' % ptr_type] \ - = ext_arguments_map.get('Column<%s>' % T) - ext_arguments_map['OmnisciColumnListType<%s>' % ptr_type] \ - = ext_arguments_map.get('ColumnList<%s>' % T) - ext_arguments_map['OmnisciOutputColumnListType<%s>' % ptr_type] \ - = ext_arguments_map.get('ColumnList<%s>' % T) - - ext_arguments_map['OmnisciBytesType'] = ext_arguments_map.get('Bytes') - - values = list(ext_arguments_map.values()) - for v, n in thrift.TExtArgumentType._VALUES_TO_NAMES.items(): - if v not in values: - warnings.warn('thrift.TExtArgumentType.%s(=%s) value not ' - 'in ext_arguments_map' % (n, v)) - self._ext_arguments_map = ext_arguments_map - return ext_arguments_map - - def type_to_extarg(self, t): - if isinstance(t, typesystem.Type): - s = t.tostring(use_annotation=False, use_name=False) - extarg = self._get_ext_arguments_map().get(s) - if extarg is None: - raise ValueError(f'cannot convert {t}(={s}) to ExtArgumentType') - return extarg - elif isinstance(t, str): - extarg = self._get_ext_arguments_map().get(t) - if extarg is None: - raise ValueError(f'cannot convert {t} to ExtArgumentType') - return extarg - elif isinstance(t, tuple): - return tuple(map(self.type_to_extarg, t)) - else: - raise TypeError(f'expected typesystem Type|tuple|str, got {type(t).__name__}') - - def retrieve_targets(self): - device_params = self.thrift_call('get_device_parameters', - self.session_id) - thrift = self.thrift_client.thrift - typemap = self.thrift_typemap - device_target_map = {} - messages = [] - for prop, value in device_params.items(): - if prop.endswith('_triple'): - device = prop.rsplit('_', 1)[0] - device_target_map[device] = value - - # Update thrift type map from server configuration - if 'Type.' in prop: - typ, member = prop.split('.', 1) - ivalue = int(value) - # Any warnings below indicate that the client and - # server thrift configurations are different. Server - # configuration will win. - if typ not in typemap: - messages.append(f'thrift type map: add new type {typ}') - elif member not in typemap[typ]: - messages.append( - f'thrift type map: add new member {typ}.{member}') - elif ivalue != typemap[typ][member]: - messages.append( - f'thrift type map: update {typ}.{member}' - f' to {ivalue} (was {typemap[typ][member]})') - else: - continue - typemap[typ][member] = ivalue - thrift_type = getattr(thrift, typ, None) - if thrift_type is not None: - thrift_type._VALUES_TO_NAMES[ivalue] = member - thrift_type._NAMES_TO_VALUES[member] = ivalue - self._ext_arguments_map = None - if messages: - warnings.warn('\n '.join([''] + messages)) - - type_sizeof = device_params.get('type_sizeof') - type_sizeof_dict = dict() - if type_sizeof is not None: - for type_size in type_sizeof.split(';'): - if not type_size: - continue - dtype, size = type_size.split(':') - type_sizeof_dict[dtype] = int(size) - - null_values = device_params.get('null_values') - null_values_asint = dict() - null_values_astype = dict() - if null_values is not None: - for tname_value in null_values.split(';'): - if not tname_value: - continue - tname, value = tname_value.split(':') - null_values_asint[tname] = int(value) - dtype = tname - if dtype.startswith('Array<'): - dtype = dtype[6:-1] - bitwidth = int(''.join(filter(str.isdigit, dtype))) - if bitwidth > 1: - null_value = numpy.dtype(f'uint{bitwidth}').type(int(value)).view( - 'int8' if dtype == 'boolean8' else dtype) - null_values_astype[tname] = null_value - self._null_values = null_values_astype - - targets = {} - for device, target in device_target_map.items(): - target_info = TargetInfo(name=device) - for prop, value in device_params.items(): - if not prop.startswith(device + '_'): - continue - target_info.set(prop[len(device)+1:], value) - if device == 'gpu': - self.has_cuda = True - # TODO: remove this hack - # see https://github.com/numba/numba/issues/4546 - target_info.set('name', 'skylake') - targets[device] = target_info - - if target_info.is_cpu: - target_info.set('driver', 'none') - target_info.add_library('m') - target_info.add_library('stdio') - target_info.add_library('stdlib') - target_info.add_library('omniscidb') - # NOTE: eventually, we want omniscidb to provide a - # 'free_buffer' function, but in the meantime we just call - # free() - target_info.set('fn_allocate_varlen_buffer', 'allocate_varlen_buffer') - target_info.set('fn_free_buffer', 'free') - elif target_info.is_gpu and self.version >= (5, 5): - target_info.add_library('libdevice') - - version_str = '.'.join(map(str, self.version[:3])) + self.version[3] - target_info.set('software', f'OmnisciDB {version_str}') - - llvm_version = device_params.get('llvm_version') - if llvm_version is not None: - target_info.set('llvm_version', tuple(map(int, llvm_version.split('.')))) - - if type_sizeof_dict is not None: - target_info.type_sizeof.update(type_sizeof_dict) - - target_info.set('null_values', null_values_asint) - return targets - - @property - def forbidden_names(self): - """Return a list of forbidden function names. See - https://github.com/xnd-project/rbc/issues/32 - """ - if self.version < (5, 2): - return ['sinh', 'cosh', 'tanh', 'rint', 'trunc', 'expm1', - 'exp2', 'log2', 'log1p', 'fmod'] - return [] - - def _make_udtf(self, caller, orig_sig, sig): - if self.version < (5, 4): - v = '.'.join(map(str, self.version)) - raise RuntimeError( - 'UDTFs with Column arguments require ' - 'omniscidb 5.4 or newer, currently ' - 'connected to ', v) - thrift = self.thrift_client.thrift - - inputArgTypes = [] - outputArgTypes = [] - sqlArgTypes = [] - annotations = [] - function_annotations = dict() # TODO: retrieve function annotations from orig_sig - sizer = None - sizer_index = -1 - - consumed_index = 0 - name = caller.func.__name__ - for i, a in enumerate(orig_sig[1]): - if is_sizer(a): - sizer = get_sizer_enum(a) - # cannot have multiple sizer arguments: - assert sizer_index == -1 - sizer_index = consumed_index + 1 - - annot = a.annotation() - - # process function annotations first to avoid appending annotations twice - if isinstance(a, OmnisciTableFunctionManagerType): - function_annotations['uses_manager'] = 'True' - consumed_index += 1 - continue - - annotations.append(annot) - - if isinstance(a, OmnisciCursorType): - sqlArgTypes.append(self.type_to_extarg('Cursor')) - for a_ in a.as_consumed_args: - assert not isinstance( - a_, OmnisciOutputColumnType), (a_) - inputArgTypes.append(self.type_to_extarg(a_)) - consumed_index += 1 - - else: - if isinstance(a, OmnisciOutputColumnType): - atype = self.type_to_extarg(a) - outputArgTypes.append(atype) - else: - atype = self.type_to_extarg(a) - if isinstance(a, (OmnisciColumnType, OmnisciColumnListType)): - sqlArgTypes.append(self.type_to_extarg('Cursor')) - inputArgTypes.append(atype) - else: - sqlArgTypes.append(atype) - inputArgTypes.append(atype) - consumed_index += 1 - if sizer is None: - sizer_index = get_literal_return(caller.func, verbose=self.debug) - if sizer_index is None: - sizer = 'kTableFunctionSpecifiedParameter' - else: - sizer = 'kConstant' - if sizer_index < 0: - raise ValueError( - f'Table function `{caller.func.__name__}`' - ' sizing parameter must be non-negative' - f' integer (got {sizer_index})') - sizer_type = (thrift.TOutputBufferSizeType - ._NAMES_TO_VALUES[sizer]) - annotations.append(function_annotations) - return thrift.TUserDefinedTableFunction( - name + sig.mangling(), - sizer_type, sizer_index, - inputArgTypes, outputArgTypes, sqlArgTypes, - annotations) - - def _make_udtf_old(self, caller, orig_sig, sig): - # old style UDTF for omniscidb <= 5.3, to be deprecated - if self.version >= (5, 4): - v = '.'.join(map(str, self.version)) - raise RuntimeError( - 'Old-style UDTFs require ' - 'omniscidb 5.3 or older, currently ' - 'connected to ', v) - thrift = self.thrift_client.thrift - - sizer = None - sizer_index = -1 - inputArgTypes = [] - outputArgTypes = [] - sqlArgTypes = [] - name = caller.func.__name__ - for i, a in enumerate(sig[1]): - _sizer = a.annotation().get('sizer') - if _sizer is not None: - # expect no more than one sizer argument - assert sizer_index == -1 - sizer_index = i + 1 - sizer = _sizer - atype = self.type_to_extarg(a) - if 'output' in a.annotation(): - outputArgTypes.append(atype) - else: - if 'input' in a.annotation(): - sqlArgTypes.append(atype) - elif 'cursor' in a.annotation(): - sqlArgTypes.append(self.type_to_extarg('Cursor')) - inputArgTypes.append(atype) - if sizer is None: - sizer = 'kConstant' - sizer_type = (thrift.TOutputBufferSizeType - ._NAMES_TO_VALUES[sizer]) - return thrift.TUserDefinedTableFunction( - name + sig.mangling(), - sizer_type, sizer_index, - inputArgTypes, outputArgTypes, sqlArgTypes) - - def _make_udf(self, caller, orig_sig, sig): - name = caller.func.__name__ - thrift = self.thrift_client.thrift - rtype = self.type_to_extarg(sig[0]) - atypes = self.type_to_extarg(sig[1]) - return thrift.TUserDefinedFunction( - name + sig.mangling(), - atypes, rtype) - - def register(self): - """Register caller cache to the server.""" - with typesystem.Type.alias(**self.typesystem_aliases): - return self._register() - - def _register(self): - if self.have_last_compile: - return - - device_ir_map = {} - llvm_function_names = [] - fid = 0 # UDF/UDTF id - udfs, udtfs = [], [] - for device, target_info in self.targets.items(): - with target_info: - udfs_map, udtfs_map = {}, {} - functions_and_signatures = [] - function_signatures = defaultdict(list) - for caller in reversed(self.get_callers()): - signatures = {} - name = caller.func.__name__ - if name in self.forbidden_names: - raise ForbiddenNameError( - f'\n\nAttempt to define function with name `{name}`.\n' - f'As a workaround, add a prefix to the function name ' - f'or define it with another name:\n\n' - f' def prefix_{name}(x):\n' - f' return np.trunc(x)\n\n' - f'For more information, see: ' - f'https://github.com/xnd-project/rbc/issues/32') - for sig in caller.get_signatures(): - i = len(function_signatures[name]) - if sig in function_signatures[name]: - if self.debug: - f2 = os.path.basename( - caller.func.__code__.co_filename) - n2 = caller.func.__code__.co_firstlineno - print(f'{type(self).__name__}.register: ignoring' - f' older definition of `{name}` for `{sig}`' - f' in {f2}#{n2}.') - continue - fid += 1 - orig_sig = sig - sig = sig[0](*sig.argument_types, **dict(name=name)) - function_signatures[name].append(sig) - sig_is_udtf = is_udtf(sig) - is_old_udtf = 'table' in sig[0].annotation() - - if i == 0 and (self.version < (5, 2) or is_old_udtf or - (self.version < (5, 5) and sig_is_udtf)): - sig.set_mangling('') - else: - if self.version < (5, 5): - sig.set_mangling('__%s' % (i)) - else: - sig.set_mangling('__%s_%s' % (device, i)) - - if sig_is_udtf: - # new style UDTF, requires omniscidb version >= 5.4 - udtfs_map[fid] = self._make_udtf(caller, orig_sig, sig) - elif is_old_udtf: - # old style UDTF for omniscidb <= 5.3, to be deprecated - udtfs_map[fid] = self._make_udtf_old(caller, orig_sig, sig) - else: - udfs_map[fid] = self._make_udf(caller, orig_sig, sig) - signatures[fid] = sig - functions_and_signatures.append((caller.func, signatures)) - - llvm_module, succesful_fids = compile_to_LLVM( - functions_and_signatures, - target_info, - pipeline_class=OmnisciCompilerPipeline, - user_defined_llvm_ir=self.user_defined_llvm_ir.get(device), - debug=self.debug) - - assert llvm_module.triple == target_info.triple - assert llvm_module.data_layout == target_info.datalayout - for f in llvm_module.functions: - llvm_function_names.append(f.name) - - device_ir_map[device] = str(llvm_module) - skipped_names = [] - for fid, udf in udfs_map.items(): - if fid in succesful_fids: - udfs.append(udf) - else: - skipped_names.append(udf.name) - - for fid, udtf in udtfs_map.items(): - if fid in succesful_fids: - udtfs.append(udtf) - else: - skipped_names.append(udtf.name) - if self.debug: - print(f'Skipping: {", ".join(skipped_names)}') - # Make sure that all registered functions have - # implementations, otherwise, we will crash the server. - for f in udtfs: - assert f.name in llvm_function_names - for f in udfs: - assert f.name in llvm_function_names - - if self.debug: - names = ", ".join([f.name for f in udfs] + [f.name for f in udtfs]) - print(f'Registering: {names}') - - self.set_last_compile(device_ir_map) - return self.thrift_call( - 'register_runtime_extension_functions', - self.session_id, udfs, udtfs, device_ir_map) - - def unregister(self): - """Unregister caller cache locally and on the server.""" - self.reset() - self.register() - - def preprocess_callable(self, func): - func = super().preprocess_callable(func) - if 'omnisci_backend' not in func.__code__.co_names: - for symbol in BufferMeta.class_names: - if symbol in func.__code__.co_names and symbol not in func.__globals__: - warnings.warn( - f'{func.__name__} uses {symbol} that may be undefined.' - f' Inserting {symbol} to global namespace.' - f' Use `from rbc.omnisci_backend import {symbol}`' - ' to remove this warning.') - func.__globals__[symbol] = omnisci_backend.__dict__.get(symbol) - return func - - _compiler = None - - @property - def compiler(self): - """Return a C++/C to LLVM IR compiler instance. - """ - if self._compiler is None: - compiler = ctools.Compiler.get(std='c++14') - if compiler is None: # clang++ not available, try clang.. - compiler = ctools.Compiler.get(std='c') - if self.debug: - print(f'compiler={compiler}') - self._compiler = compiler - return self._compiler - - def caller_signature(self, signature: typesystem.Type): - """Return signature of a caller. - - See RemoteJIT.caller_signature.__doc__. - """ - if is_udtf(signature): - rtype = signature[0] - if not (rtype.is_int and rtype.bits == 32): - raise ValueError( - f'UDTF implementation return type must be int32, got {rtype}') - rtypes = [] - atypes = [] - for atype in signature[1]: - if is_sizer(atype): - sizer = get_sizer_enum(atype) - if sizer not in user_specified_output_buffer_sizers: - continue - atype.annotation(sizer=sizer) - elif isinstance(atype, OmnisciTableFunctionManagerType): - continue - elif isinstance(atype, OmnisciOutputColumnType): - rtypes.append(atype.copy(OmnisciColumnType)) - continue - atypes.append(atype) - rtype = typesystem.Type(*rtypes, **dict(struct_is_tuple=True)) - return rtype(*atypes, **signature._params) - return signature - - def get_types(self, *values): - """Convert values to the corresponding typesystem types. - - See RemoteJIT.get_types.__doc__. - """ - types = [] - for value in values: - if isinstance(value, RemoteCallCapsule): - typ = value.__typesystem_type__ - if typ.is_struct and typ._params.get('struct_is_tuple'): - types.extend(typ) - else: - types.append(typ) - else: - types.append(typesystem.Type.fromvalue(value)) - return tuple(types) - - def normalize_function_type(self, ftype: typesystem.Type): - """Normalize typesystem function type. - - See RemoteJIT.normalize_function_type.__doc__. - """ - assert ftype.is_function, ftype - for atype in ftype[1]: - # convert `T foo` to `T | name=foo` - if 'name' not in atype.annotation() and atype.name is not None: - atype.annotation(name=atype.name) - atype._params.pop('name') - return ftype - - def format_type(self, typ: typesystem.Type): - """Convert typesystem type to formatted string. - - See RemoteJIT.format_type.__doc__. - """ - if typ.is_function: - args = map(self.format_type, typ[1]) - if is_udtf(typ): - return f'UDTF({", ".join(args)})' - else: - return f'({", ".join(args)}) -> {self.format_type(typ[0])}' - use_typename = False - if typ.is_struct and typ._params.get('struct_is_tuple'): - return f'({", ".join(map(self.format_type, typ))})' - if isinstance(typ, OmnisciOutputColumnType): - p = tuple(map(self.format_type, typ[0])) - typ = typesystem.Type(('OutputColumn',) + p, **typ._params) - elif isinstance(typ, OmnisciColumnType): - p = tuple(map(self.format_type, typ[0])) - typ = typesystem.Type(('Column',) + p, **typ._params) - elif isinstance(typ, OmnisciColumnListType): - p = tuple(map(self.format_type, typ[0])) - typ = typesystem.Type(('ColumnList',) + p, **typ._params) - elif isinstance(typ, OmnisciArrayType): - p = tuple(map(self.format_type, typ[0])) - typ = typesystem.Type(('Array',) + p, **typ._params) - elif isinstance(typ, OmnisciCursorType): - p = tuple(map(self.format_type, typ[0])) - typ = typesystem.Type(('Cursor',) + p, **typ._params) - elif isinstance(typ, OmnisciBytesType): - typ = typ.copy().params(typename='Bytes') - use_typename = True - elif isinstance(typ, OmnisciTextEncodingDictType): - typ = typ.copy().params(typename='TextEncodingDict') - use_typename = True - elif isinstance(typ, OmnisciTableFunctionManagerType): - typ = typ.copy().params(typename='TableFunctionManager') - use_typename = True - elif is_sizer(typ): - sizer = get_sizer_enum(typ) - for shortname, fullname in output_buffer_sizer_map.items(): - if fullname == sizer: - use_typename = True - typ = typ.copy().params(typename=shortname) - typ.annotation().pop('sizer') - break - - return typ.tostring(use_typename=use_typename, use_annotation_name=True) - - # We define remote_compile and remote_call for Caller.__call__ method. - def remote_compile(self, func, ftype: typesystem.Type, target_info: TargetInfo): - """Remote compile function and signatures to machine code. - - See RemoteJIT.remote_compile.__doc__. - """ - if self.query_requires_register(func.__name__): - self.register() - - def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False): - """ - See RemoteJIT.remote_call.__doc__. - """ - sig = self.caller_signature(ftype) - assert len(arguments) == len(sig[1]), (sig, arguments) - rtype = sig[0] - args = [] - for a, atype in zip(arguments, sig[1]): - if isinstance(a, RemoteCallCapsule): - if is_udtf(a.ftype): - a = a.execute(hold=True) - else: - a = a.execute(hold=True).lstrip('SELECT ') - - if isinstance(atype, (OmnisciColumnType, OmnisciColumnListType)): - args.append(f'CURSOR({a})') - elif isinstance(atype, OmnisciBytesType): - if isinstance(a, bytes): - a = repr(a.decode()) - elif isinstance(a, str): - a = repr(a) - args.append(f'{a}') - else: - args.append(f'CAST({a} AS {type_to_type_name(atype)})') - args = ', '.join(args) - is_udtf_call = is_udtf(ftype) - if is_udtf_call: - colnames = [] - if rtype.is_struct and rtype._params.get('struct_is_tuple'): - for i, t in enumerate(rtype): - n = t.annotation().get('name', f'out{i}') - colnames.append(n) - else: - colnames.append(rtype.annotation().get('name', '*')) - q = f'SELECT {", ".join(colnames)} FROM TABLE({func.__name__}({args}))' - else: - q = f'SELECT {func.__name__}({args})' - if hold: - return q - descrs, result = self.sql_execute(q + ';') - dtype = [(descr.name, type_name_to_dtype(descr.type_name)) for descr in descrs] - if is_udtf_call: - return numpy.array(list(result), dtype).view(numpy.recarray) - else: - return dtype[0][1](list(result)[0][0]) - - -class RemoteOmnisci(RemoteHeavyDB): - """Omnisci - the previous brand of HeavyAI - """ +msg = "`import rbc.omniscidb` is deprecated, use `import rbc.heavyai` instead." +warnings.warn(msg, PendingDeprecationWarning) diff --git a/rbc/remotejit.py b/rbc/remotejit.py index dc03d7ab..3e245051 100644 --- a/rbc/remotejit.py +++ b/rbc/remotejit.py @@ -19,7 +19,7 @@ # XXX WIP: the OmnisciCompilerPipeline is no longer omnisci-specific because # we support Arrays even without omnisci, so it must be renamed and moved # somewhere elsef -from .omnisci_backend import OmnisciCompilerPipeline +from .heavyai import OmnisciCompilerPipeline def isfunctionlike(obj): diff --git a/rbc/stdlib/__init__.py b/rbc/stdlib/__init__.py index 10fed296..f09f6266 100644 --- a/rbc/stdlib/__init__.py +++ b/rbc/stdlib/__init__.py @@ -1,7 +1,7 @@ import functools from enum import Enum from numba.core import extending -from rbc.omnisci_backend import Array, ArrayPointer +from rbc.heavyai import Array, ArrayPointer from rbc import typesystem, errors diff --git a/rbc/stdlib/creation_functions.py b/rbc/stdlib/creation_functions.py index 91d7bb61..81fd9e07 100644 --- a/rbc/stdlib/creation_functions.py +++ b/rbc/stdlib/creation_functions.py @@ -5,7 +5,7 @@ """ from rbc import typesystem -from rbc.omnisci_backend.omnisci_array import Array, ArrayPointer +from rbc.heavyai.array import Array, ArrayPointer from rbc.stdlib import Expose from numba import njit from numba.core import extending, types diff --git a/rbc/stdlib/datatypes.py b/rbc/stdlib/datatypes.py index b7090a5e..e3de33c4 100644 --- a/rbc/stdlib/datatypes.py +++ b/rbc/stdlib/datatypes.py @@ -19,10 +19,10 @@ 'float64' ] -# NOTE: currently the code lives in rbc.omnisci_backend, but eventually we -# should move it here and leave rbc.omnisci_backend.Array only for backwards +# NOTE: currently the code lives in rbc.heavyai, but eventually we +# should move it here and leave rbc.heavyai.Array only for backwards # compatibility -from rbc.omnisci_backend import Array +from rbc.heavyai import Array # array API data types from numba.types import ( diff --git a/rbc/stdlib/elementwise_functions.py b/rbc/stdlib/elementwise_functions.py index 5bc415d2..063ac937 100644 --- a/rbc/stdlib/elementwise_functions.py +++ b/rbc/stdlib/elementwise_functions.py @@ -7,7 +7,7 @@ from rbc.stdlib import Expose, BinaryUfuncExpose, UnaryUfuncExpose, API, determine_input_type import numpy as np from rbc import typesystem -from rbc.omnisci_backend import ArrayPointer, Array +from rbc.heavyai import ArrayPointer, Array from numba.core import types diff --git a/rbc/stdlib/statistical_functions.py b/rbc/stdlib/statistical_functions.py index 1bba880f..b31e5ea6 100644 --- a/rbc/stdlib/statistical_functions.py +++ b/rbc/stdlib/statistical_functions.py @@ -5,7 +5,7 @@ """ from rbc.externals.stdio import printf from rbc import typesystem -from rbc.omnisci_backend import ArrayPointer +from rbc.heavyai import ArrayPointer from rbc.stdlib import Expose from numba.core import extending, types, errors from numba.np import numpy_support diff --git a/rbc/tests/__init__.py b/rbc/tests/__init__.py index 08c3869f..e3cb9768 100644 --- a/rbc/tests/__init__.py +++ b/rbc/tests/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['omnisci_fixture', 'sql_execute'] +__all__ = ['heavydb_fixture', 'sql_execute'] import os @@ -23,16 +23,16 @@ def assert_equal(actual, desired): def sql_execute(query): - """Execute a SQL statement to omniscidb server using global instance. + """Execute a SQL statement to heavydb server using global instance. Use when the query does not require registration of new UDF/UDTFs. """ - rbc_omnisci = pytest.importorskip('rbc.omniscidb') - omnisci = next(rbc_omnisci.global_omnisci_singleton) - return omnisci.sql_execute(query) + rbc_heavydb = pytest.importorskip('rbc.heavydb') + heavydb = next(rbc_heavydb.global_heavydb_singleton) + return heavydb.sql_execute(query) -def omnisci_fixture(caller_globals, minimal_version=(0, 0), +def heavydb_fixture(caller_globals, minimal_version=(0, 0), suffices=['', '10', 'null', 'array', 'arraynull'], load_columnar=True, load_test_data=True, debug=False): """Usage from a rbc/tests/test_xyz.py file: @@ -40,80 +40,79 @@ def omnisci_fixture(caller_globals, minimal_version=(0, 0), .. code-block:: python import pytest - from rbc.tests import omnisci_fixture + from rbc.tests import heavydb_fixture @pytest.fixture(scope='module') def omnisci(): - from o in omnisci_fixture(globals()): + from o in heavydb_fixture(globals()): # do some customization here yield o This fixture creates the following tables: - f'{omnisci.table_name}' - contains columns f8, f4, i8, i4, i2, i1, + f'{heavydb.table_name}' - contains columns f8, f4, i8, i4, i2, i1, b with row size 5. - f'{omnisci.table_name}10' - contains columns f8, f4, i8, i4, i2, + f'{heavydb.table_name}10' - contains columns f8, f4, i8, i4, i2, i1, b with row size 10. - f'{omnisci.table_name}null' - contains columns f8, f4, i8, i4, i2, + f'{heavydb.table_name}null' - contains columns f8, f4, i8, i4, i2, i1, b with row size 5, contains null values - f'{omnisci.table_name}array' - contains arrays f8, f4, i8, i4, i2, + f'{heavydb.table_name}array' - contains arrays f8, f4, i8, i4, i2, i1, b with row size 5 - f'{omnisci.table_name}arraynull' - contains arrays f8, f4, i8, i4, i2, + f'{heavydb.table_name}arraynull' - contains arrays f8, f4, i8, i4, i2, i1, b with row size 5, contains null values. """ - rbc_omnisci = pytest.importorskip('rbc.omniscidb') - available_version, reason = rbc_omnisci.is_available() + rbc_heavydb = pytest.importorskip('rbc.heavydb') + available_version, reason = rbc_heavydb.is_available() def require_version(version, message=None, label=None): """Execute pytest.skip(...) if version is older than available_version. - Some tests can be run only when using omniscidb server built - from a particular branch of omniscidb. So, when the specified - version and the omniscidb version match exactly and these + Some tests can be run only when using heavydb server built + from a particular branch of heavydb. So, when the specified + version and the heavydb version match exactly and these correspond to the current development version, if the specified label does not match with the value of envrinment variable OMNISCIDB_DEV_LABEL, then the corresponing test will - be skipped. Use label 'docker-dev' when using omniscidb dev + be skipped. Use label 'docker-dev' when using heavydb dev docker image. """ - # The available version (of the omniscidb server) has date and + # The available version (of the heavydb server) has date and # hash bits, however, these are useless for determining the # version ordering (in the case of available_version[:3] == # version) because the date corresponds to the date of # building the server and the hash corresponds to some commit - # of some repository (omniscidb or omniscidb-internal) and it + # of some repository (heavydb or heavydb-internal) and it # does not provide easy date information. # # The condition available_version[:3] == version can appear in # the following cases (given in the order of from newer to # older): - # 1. omniscidb is built against a omniscidb-internal PR branch + # 1. heavydb is built against a heavydb-internal PR branch # (assuming it is rebased against master) - # 2. omniscidb is built against omniscidb-internal master branch - # 3. omniscidb is built against omniscidb master branch - # 4. omniscidb is built against omniscidb dev docker - # 5. omniscidb is built against omniscidb/omniscidb-internal release tag + # 2. heavydb is built against heavydb-internal master branch + # 3. heavydb is built against heavydb master branch + # 4. heavydb is built against heavydb dev docker + # 5. heavydb is built against heavydb/heavydb-internal release tag # # rbc testing suite may use features that exists in the head # of the above list but not in the tail of it. So we have a # problem of deciding if a particular test should be disabled # or not for a given case while there is no reliable way to - # tell from omniscidb version if the server has the particular + # tell from heavydb version if the server has the particular # feature or not. To resolve this, we use label concept as # explained in the doc-string. # if not available_version: pytest.skip(reason) - - # Requires update when omniscidb-internal bumps up version number: + # Requires update when heavydb-internal bumps up version number: current_development_version = (6, 1, 0) if available_version[:3] > current_development_version: warnings.warn(f'{available_version}) is newer than development version' @@ -163,8 +162,8 @@ def require_version(version, message=None, label=None): filename = caller_globals['__file__'] table_name = os.path.splitext(os.path.basename(filename))[0] - config = rbc_omnisci.get_client_config(debug=debug) - m = rbc_omnisci.RemoteOmnisci(**config) + config = rbc_heavydb.get_client_config(debug=debug) + m = rbc_heavydb.RemoteHeavyDB(**config) if not load_test_data: yield m @@ -176,7 +175,7 @@ def require_version(version, message=None, label=None): # todo: TEXT ENCODING DICT, TEXT ENCODING NONE, TIMESTAMP, TIME, # DATE, DECIMAL/NUMERIC, GEOMETRY: POINT, LINESTRING, POLYGON, # MULTIPOLYGON, See - # https://www.omnisci.com/docs/latest/5_datatypes.html + # https://docs.heavy.ai/sql/data-definition-ddl/datatypes-and-fixed-encoding colnames = ['f4', 'f8', 'i1', 'i2', 'i4', 'i8', 'b'] table_defn = ',\n'.join('%s %s' % (n, t) for t, n in zip(sqltypes, colnames)) diff --git a/rbc/tests/heavyai/__init__.py b/rbc/tests/heavyai/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rbc/tests/test_omnisci_array.py b/rbc/tests/heavyai/test_array.py similarity index 61% rename from rbc/tests/test_omnisci_array.py rename to rbc/tests/heavyai/test_array.py index 30eeca8d..f29eac93 100644 --- a/rbc/tests/test_omnisci_array.py +++ b/rbc/tests/heavyai/test_array.py @@ -1,21 +1,22 @@ import os from collections import defaultdict -from rbc.omnisci_backend import Array + +from rbc.heavyai import Array from rbc.errors import OmnisciServerError from rbc.stdlib import array_api from numba import types as nb_types import pytest -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) table_name = os.path.splitext(os.path.basename(__file__))[0] m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') @@ -25,7 +26,7 @@ def omnisci(): # todo: TEXT ENCODING DICT, TEXT ENCODING NONE, TIMESTAMP, TIME, # DATE, DECIMAL/NUMERIC, GEOMETRY: POINT, LINESTRING, POLYGON, # MULTIPOLYGON, See - # https://www.omnisci.com/docs/latest/5_datatypes.html + # https://docs.heavy.ai/sql/data-definition-ddl/datatypes-and-fixed-encoding colnames = ['f4', 'f8', 'i1', 'i2', 'i4', 'i8', 'b'] table_defn = ',\n'.join('%s %s' % (n, t) for t, n in zip(sqltypes, colnames)) @@ -53,13 +54,13 @@ def omnisci(): @pytest.mark.parametrize('c_name', ['int8_t i1', 'int16_t i2', 'int32_t i4', 'int64_t i8', 'float f4', 'double f8']) @pytest.mark.parametrize('device', ['cpu', 'gpu']) -def test_ptr(omnisci, c_name, device): - omnisci.reset() - if not omnisci.has_cuda and device == 'gpu': - pytest.skip('test requires CUDA-enabled omniscidb server') +def test_ptr(heavydb, c_name, device): + heavydb.reset() + if not heavydb.has_cuda and device == 'gpu': + pytest.skip('test requires CUDA-enabled heavydb server') from rbc.external import external - if omnisci.compiler is None: + if heavydb.compiler is None: pytest.skip('test requires clang C/C++ compiler') ctype, cname = c_name.split() @@ -84,141 +85,141 @@ def test_ptr(omnisci, c_name, device): }} #endif ''' - omnisci.user_defined_llvm_ir[device] = omnisci.compiler(c_code) + heavydb.user_defined_llvm_ir[device] = heavydb.compiler(c_code) mysum_impl = external(f'{ctype} mysum_impl({ctype}*, int32_t)') myval_impl = external(f'{ctype} myval_impl({ctype}*)') - @omnisci(f'{ctype}({ctype}[])', devices=[device]) + @heavydb(f'{ctype}({ctype}[])', devices=[device]) def mysum_ptr(x): return mysum_impl(x.ptr(), len(x)) - @omnisci(f'{ctype}({ctype}[], int32_t)', devices=[device]) + @heavydb(f'{ctype}({ctype}[], int32_t)', devices=[device]) def myval_ptr(x, i): return myval_impl(x.ptr(i)) - desrc, result = omnisci.sql_execute( - f'select {cname}, mysum_ptr({cname}) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select {cname}, mysum_ptr({cname}) from {heavydb.table_name}') for a, r in result: if cname == 'i1': assert sum(a) % 256 == r % 256 else: assert sum(a) == r - desrc, result = omnisci.sql_execute( - f'select {cname}, myval_ptr({cname}, 0), myval_ptr({cname}, 2) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select {cname}, myval_ptr({cname}, 0), myval_ptr({cname}, 2) from {heavydb.table_name}') for a, r0, r2 in result: assert a[0] == r0 assert a[2] == r2 -def test_len_i32(omnisci): - omnisci.reset() +def test_len_i32(heavydb): + heavydb.reset() - @omnisci('int64(int32[])') + @heavydb('int64(int32[])') def array_sz_int32(x): return len(x) - desrc, result = omnisci.sql_execute( - f'select i4, array_sz_int32(i4) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select i4, array_sz_int32(i4) from {heavydb.table_name}') for a, sz in result: assert len(a) == sz -def test_len_f64(omnisci): - omnisci.reset() +def test_len_f64(heavydb): + heavydb.reset() - @omnisci('int64(float64[])') + @heavydb('int64(float64[])') def array_sz_double(x): return len(x) - desrc, result = omnisci.sql_execute( - f'select f8, array_sz_double(f8) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select f8, array_sz_double(f8) from {heavydb.table_name}') for a, sz in result: assert len(a) == sz @pytest.mark.skipif(available_version[:2] == (5, 1), - reason="skip due to a bug in omniscidb 5.1 (got %s)" % ( + reason="skip due to a bug in heavydb 5.1 (got %s)" % ( available_version,)) -def test_getitem_bool(omnisci): - omnisci.reset() +def test_getitem_bool(heavydb): + heavydb.reset() - @omnisci('bool(bool[], int64)') + @heavydb('bool(bool[], int64)') def array_getitem_bool(x, i): return x[i] - query = f'select b, array_getitem_bool(b, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select b, array_getitem_bool(b, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item -def test_getitem_i8(omnisci): - omnisci.reset() +def test_getitem_i8(heavydb): + heavydb.reset() - @omnisci('int8(int8[], int32)') + @heavydb('int8(int8[], int32)') def array_getitem_int8(x, i): return x[i] - query = f'select i1, array_getitem_int8(i1, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select i1, array_getitem_int8(i1, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item -def test_getitem_i32(omnisci): - omnisci.reset() +def test_getitem_i32(heavydb): + heavydb.reset() - @omnisci('int32(int32[], int32)') + @heavydb('int32(int32[], int32)') def array_getitem_int32(x, i): return x[i] - query = f'select i4, array_getitem_int32(i4, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select i4, array_getitem_int32(i4, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item -def test_getitem_i64(omnisci): - omnisci.reset() +def test_getitem_i64(heavydb): + heavydb.reset() - @omnisci('int64(int64[], int64)') + @heavydb('int64(int64[], int64)') def array_getitem_int64(x, i): return x[i] - query = f'select i8, array_getitem_int64(i8, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select i8, array_getitem_int64(i8, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item -def test_getitem_float(omnisci): - omnisci.reset() +def test_getitem_float(heavydb): + heavydb.reset() - @omnisci('double(double[], int32)') + @heavydb('double(double[], int32)') def array_getitem_double(x, i): return x[i] - query = f'select f8, array_getitem_double(f8, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select f8, array_getitem_double(f8, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item assert type(a[2]) == type(item) - @omnisci('float(float[], int64)') + @heavydb('float(float[], int64)') def array_getitem_float(x, i): return x[i] - query = f'select f4, array_getitem_float(f4, 2) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select f4, array_getitem_float(f4, 2) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, item in result: assert a[2] == item assert type(a[2]) == type(item) -def test_sum(omnisci): - omnisci.reset() +def test_sum(heavydb): + heavydb.reset() - @omnisci('int32(int32[])') + @heavydb('int32(int32[])') def array_sum_int32(x): r = 0 n = len(x) @@ -226,19 +227,19 @@ def array_sum_int32(x): r = r + x[i] return r - query = f'select i4, array_sum_int32(i4) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select i4, array_sum_int32(i4) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for a, s in result: assert sum(a) == s @pytest.mark.skipif(available_version[:2] == (5, 1), - reason="skip due to a bug in omniscidb 5.1 (got %s)" % ( + reason="skip due to a bug in heavydb 5.1 (got %s)" % ( available_version,)) -def test_even_sum(omnisci): - omnisci.reset() +def test_even_sum(heavydb): + heavydb.reset() - @omnisci('int32(bool[], int32[])') + @heavydb('int32(bool[], int32[])') def array_even_sum_int32(b, x): r = 0 n = len(x) @@ -247,16 +248,16 @@ def array_even_sum_int32(b, x): r = r + x[i] return r - query = f'select b, i4, array_even_sum_int32(b, i4) from {omnisci.table_name}' - desrc, result = omnisci.sql_execute(query) + query = f'select b, i4, array_even_sum_int32(b, i4) from {heavydb.table_name}' + desrc, result = heavydb.sql_execute(query) for b, i4, s in result: assert sum([i_ for b_, i_ in zip(b, i4) if b_]) == s -def test_array_setitem(omnisci): - omnisci.reset() +def test_array_setitem(heavydb): + heavydb.reset() - @omnisci('double(double[], int32)') + @heavydb('double(double[], int32)') def array_setitem_sum(b, c): n = len(b) s = 0 @@ -266,19 +267,19 @@ def array_setitem_sum(b, c): b[i] = b[i] / c return s - query = f'select f8, array_setitem_sum(f8, 4) from {omnisci.table_name}' - _, result = omnisci.sql_execute(query) + query = f'select f8, array_setitem_sum(f8, 4) from {heavydb.table_name}' + _, result = heavydb.sql_execute(query) for f8, s in result: assert sum(f8) * 4 == s -def test_array_constructor_noreturn(omnisci): - omnisci.reset() +def test_array_constructor_noreturn(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Array + from rbc.heavyai import Array - @omnisci('float64(int32)') + @heavydb('float64(int32)') def array_noreturn(size): a = Array(size, nb_types.float64) b = Array(size, nb_types.float64) @@ -291,18 +292,18 @@ def array_noreturn(size): return s query = 'select array_noreturn(10)' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) r = list(result)[0] assert (r == (-420.0,)) -def test_array_constructor_return(omnisci): - omnisci.reset() +def test_array_constructor_return(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Array + from rbc.heavyai import Array from rbc.externals.stdio import printf - @omnisci('float64[](int32)') + @heavydb('float64[](int32)') def array_return(size): printf("entering array_return(%i)\n", size) a = Array(size, nb_types.float64) @@ -318,36 +319,36 @@ def array_return(size): return c query = 'select array_return(9), array_return(10)' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) r = list(result)[0] assert r == (list(map(float, range(9))), list(map(float, reversed(range(10))))) -def test_array_constructor_len(omnisci): - omnisci.reset() +def test_array_constructor_len(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Array + from rbc.heavyai import Array - @omnisci('int64(int32)') + @heavydb('int64(int32)') def array_len(size): a = Array(size, nb_types.float64) return len(a) query = 'select array_len(30)' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result)[0] == (30,) -def test_array_constructor_getitem(omnisci): - omnisci.reset() +def test_array_constructor_getitem(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Array + from rbc.heavyai import Array import numpy as np - @omnisci('double(int32, int32)') + @heavydb('double(int32, int32)') def array_ptr(size, pos): a = Array(size, np.double) for i in range(size): @@ -355,23 +356,23 @@ def array_ptr(size, pos): return a[pos] query = 'select array_ptr(5, 3)' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result)[0] == (3.0,) -def test_array_constructor_is_null(omnisci): - omnisci.reset() +def test_array_constructor_is_null(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Array + from rbc.heavyai import Array - @omnisci('int8(int64)') + @heavydb('int8(int64)') def array_is_null(size): a = Array(size, 'double') return a.is_null() query = 'select array_is_null(3);' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result)[0] == (0,) @@ -383,8 +384,8 @@ def array_is_null(size): @pytest.mark.parametrize("typ, col, suffix", inps, ids=[item[-1] for item in inps]) -def test_issue197(omnisci, typ, col, suffix): - omnisci.reset() +def test_issue197(heavydb, typ, col, suffix): + heavydb.reset() cast = dict( trunc=nb_types.int64, @@ -402,10 +403,10 @@ def fn_issue197(x): fn_name = f"fn_issue197_{typ}_{suffix}" fn_issue197.__name__ = fn_name - omnisci(f'{typ}[]({typ}[])')(fn_issue197) + heavydb(f'{typ}[]({typ}[])')(fn_issue197) - _, result = omnisci.sql_execute( - f'SELECT {col}, {fn_name}({col}) FROM {omnisci.table_name};' + _, result = heavydb.sql_execute( + f'SELECT {col}, {fn_name}({col}) FROM {heavydb.table_name};' ) column, ret = list(result)[0] @@ -413,10 +414,10 @@ def fn_issue197(x): assert y == x + 3 -def test_issue197_bool(omnisci): - omnisci.reset() +def test_issue197_bool(heavydb): + heavydb.reset() - @omnisci('bool[](bool[])') + @heavydb('bool[](bool[])') def fn_issue197_bool(x): y = array_api.zeros_like(x) for i in range(len(x)): @@ -426,8 +427,8 @@ def fn_issue197_bool(x): col = 'b' fn_name = 'fn_issue197_bool' - _, result = omnisci.sql_execute( - f'SELECT {col}, {fn_name}({col}) FROM {omnisci.table_name};' + _, result = heavydb.sql_execute( + f'SELECT {col}, {fn_name}({col}) FROM {heavydb.table_name};' ) column, ret = list(result)[0] @@ -435,41 +436,41 @@ def fn_issue197_bool(x): assert bool(x) == bool(y) -def test_issue109(omnisci): +def test_issue109(heavydb): - @omnisci('double[](int32)') + @heavydb('double[](int32)') def issue109(size): a = Array(5, 'double') for i in range(5): a[i] = nb_types.double(i) return a - _, result = omnisci.sql_execute('select issue109(3);') + _, result = heavydb.sql_execute('select issue109(3);') assert list(result) == [([0.0, 1.0, 2.0, 3.0, 4.0],)] -def test_issue77(omnisci): +def test_issue77(heavydb): - @omnisci('int64[]()') + @heavydb('int64[]()') def issue77(): a = Array(5, 'int64') a.fill(1) return a - if omnisci.version[:2] >= (5, 8): - _, result = omnisci.sql_execute('select issue77();') + if heavydb.version[:2] >= (5, 8): + _, result = heavydb.sql_execute('select issue77();') assert list(result)[0][0] == [1, 1, 1, 1, 1] else: with pytest.raises(OmnisciServerError) as exc: - _, result = omnisci.sql_execute('select issue77();') + _, result = heavydb.sql_execute('select issue77();') assert exc.match('Could not bind issue77()') -def test_array_dtype(omnisci): - table = omnisci.table_name +def test_array_dtype(heavydb): + table = heavydb.table_name - @omnisci('T(T[])', T=['int32', 'int64']) + @heavydb('T(T[])', T=['int32', 'int64']) def array_dtype_fn(x): if x.dtype == nb_types.int32: return 32 @@ -477,20 +478,20 @@ def array_dtype_fn(x): return 64 for col, r in (('i4', 32), ('i8', 64)): - _, result = omnisci.sql_execute(f'select array_dtype_fn({col}) from {table}') + _, result = heavydb.sql_execute(f'select array_dtype_fn({col}) from {table}') assert list(result) == [(r,)] * 5 -def test_array_enumerate(omnisci): - table = omnisci.table_name +def test_array_enumerate(heavydb): + table = heavydb.table_name - @omnisci('T(T[])', T=['int32']) + @heavydb('T(T[])', T=['int32']) def array_enumerate(x): s = 0 for i, e in enumerate(x): s += e return s - _, result = omnisci.sql_execute(f'select i4, array_enumerate(i4) from {table}') + _, result = heavydb.sql_execute(f'select i4, array_enumerate(i4) from {table}') for arr, s in result: assert sum(arr) == s diff --git a/rbc/tests/test_array_api_unsupported.py b/rbc/tests/heavyai/test_array_api_unsupported.py similarity index 85% rename from rbc/tests/test_array_api_unsupported.py rename to rbc/tests/heavyai/test_array_api_unsupported.py index aed1308e..2d58660e 100644 --- a/rbc/tests/test_array_api_unsupported.py +++ b/rbc/tests/heavyai/test_array_api_unsupported.py @@ -1,13 +1,13 @@ import pytest from rbc.stdlib import array_api -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from numba import TypingError @pytest.fixture(scope="module") -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals()): + for o in heavydb_fixture(globals()): yield o @@ -37,14 +37,16 @@ def omnisci(): # ensure unimplemented functions raise a meaninful exception @pytest.mark.parametrize('func_name', unsupported_functions) -def test_unimplemented(omnisci, func_name): +@pytest.mark.skip() +def test_unimplemented(heavydb, func_name): func = getattr(array_api, func_name) - @omnisci('int64(int64)') + @heavydb('int64(int64)') def test_exception_raised(x): return func(x) # NumbaNotSupportedError is captured and a TypingError is returned instead with pytest.raises(TypingError, match=f'Function "{func_name}" is not supported.'): - omnisci.register() + heavydb.register() + heavydb.unregister() diff --git a/rbc/tests/test_omnisci_array_functions.py b/rbc/tests/heavyai/test_array_functions.py similarity index 84% rename from rbc/tests/test_omnisci_array_functions.py rename to rbc/tests/heavyai/test_array_functions.py index 82e2c65f..df37fe13 100644 --- a/rbc/tests/test_omnisci_array_functions.py +++ b/rbc/tests/heavyai/test_array_functions.py @@ -1,20 +1,21 @@ import pytest import numpy as np + from rbc.stdlib import array_api from numba.core import types -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) - table_name = 'rbc_test_omnisci_array' +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) + table_name = 'rbc_test_heavydb_array' m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') sqltypes = ['FLOAT[]', 'DOUBLE[]', @@ -23,7 +24,7 @@ def omnisci(): # todo: TEXT ENCODING DICT, TEXT ENCODING NONE, TIMESTAMP, TIME, # DATE, DECIMAL/NUMERIC, GEOMETRY: POINT, LINESTRING, POLYGON, # MULTIPOLYGON, See - # https://www.omnisci.com/docs/latest/5_datatypes.html + # https://docs.heavy.ai/sql/data-definition-ddl/datatypes-and-fixed-encoding colnames = ['f4', 'f8', 'i1', 'i2', 'i4', 'i8', 'b'] table_defn = ',\n'.join('%s %s' % (n, t) for t, n in zip(sqltypes, colnames)) @@ -142,16 +143,16 @@ def np_cumsum(sz): @pytest.mark.parametrize("method, signature, args, expected", array_methods, ids=[item[0] for item in array_methods]) -def test_array_methods(omnisci, method, signature, args, expected): - omnisci.reset() +def test_array_methods(heavydb, method, signature, args, expected): + heavydb.reset() - fn = omnisci(signature)(eval('np_{}'.format(method))) + fn = heavydb(signature)(eval('np_{}'.format(method))) query = 'select np_{method}'.format(**locals()) + \ '(' + ', '.join(map(str, args)) + ')' + \ - ' from {omnisci.table_name};'.format(**locals()) + ' from {heavydb.table_name};'.format(**locals()) - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) out = list(result)[0] if 'empty' in method: @@ -161,14 +162,14 @@ def test_array_methods(omnisci, method, signature, args, expected): @pytest.mark.parametrize('col', ('i4', 'i8', 'f4')) -def test_dtype(omnisci, col): - omnisci.reset() +def test_dtype(heavydb, col): + heavydb.reset() - @omnisci('T[](T[])', T=['int32', 'int64', 'float32'], devices=['cpu']) + @heavydb('T[](T[])', T=['int32', 'int64', 'float32'], devices=['cpu']) def zeros_like(x): z = array_api.zeros(len(x), x.dtype) return z - query = f'select zeros_like({col}) from {omnisci.table_name} limit 1;' - _, result = omnisci.sql_execute(query) + query = f'select zeros_like({col}) from {heavydb.table_name} limit 1;' + _, result = heavydb.sql_execute(query) assert np.all(list(result)[0][0] == np.zeros(6, dtype=col)) diff --git a/rbc/tests/test_omnisci_array_math.py b/rbc/tests/heavyai/test_array_math.py similarity index 88% rename from rbc/tests/test_omnisci_array_math.py rename to rbc/tests/heavyai/test_array_math.py index 554ce5f9..7250273f 100644 --- a/rbc/tests/test_omnisci_array_math.py +++ b/rbc/tests/heavyai/test_array_math.py @@ -1,20 +1,20 @@ import pytest import numpy as np -import rbc.omnisci_backend as omni # noqa: F401 +import rbc.heavyai as omni # noqa: F401 from rbc.stdlib import array_api -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) - table_name = 'rbc_test_omnisci_array' +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) + table_name = 'rbc_test_heavydb_array' m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') sqltypes = ['FLOAT[]', 'DOUBLE[]', @@ -23,7 +23,7 @@ def omnisci(): # todo: TEXT ENCODING DICT, TEXT ENCODING NONE, TIMESTAMP, TIME, # DATE, DECIMAL/NUMERIC, GEOMETRY: POINT, LINESTRING, POLYGON, # MULTIPOLYGON, See - # https://www.omnisci.com/docs/latest/5_datatypes.html + # https://www.heavydb.com/docs/latest/5_datatypes.html colnames = ['f4', 'f8', 'i1', 'i2', 'i4', 'i8', 'b', 'u8'] table_defn = ',\n'.join('%s %s' % (n, t) for t, n in zip(sqltypes, colnames)) @@ -111,21 +111,21 @@ def is_number(s): @pytest.mark.parametrize("method, signature, columns", binary_fns, ids=[item[0] for item in binary_fns]) -def test_omnisci_array_binary_math(omnisci, method, signature, columns): - omnisci.reset() +def test_heavydb_array_binary_math(heavydb, method, signature, columns): + heavydb.reset() s = f'def np_{method}(a, b): return array_api.{method}(a, b)' exec(s, globals()) - omnisci(signature)(eval('np_{}'.format(method))) + heavydb(signature)(eval('np_{}'.format(method))) ca, cb = columns.split(',') query = f'select {ca}, {cb}, ' + \ f'np_{method}({ca}, {cb})' + \ - f' from {omnisci.table_name};' + f' from {heavydb.table_name};' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) row_a, row_b, out = list(result)[0] expected = getattr(np, method)(row_a, row_b) @@ -135,7 +135,7 @@ def test_omnisci_array_binary_math(omnisci, method, signature, columns): binary_fn_scalar_input = [ ('add', 'int32[](int32[], int32)', 'i4,3'), ('subtract', 'double[](double[], double)', 'f8,5.0'), - # omnisci server crashes: + # heavydb server crashes: # ('subtract', 'double[](double, double[])', '5.0,f8'), ('multiply', 'double[](double[], double)', 'f8,5.0'), ('divide', 'double[](double[], double)', 'f8,2.0'), @@ -173,13 +173,13 @@ def test_omnisci_array_binary_math(omnisci, method, signature, columns): @pytest.mark.parametrize("method, signature, args", binary_fn_scalar_input, ids=[item[0] for item in binary_fn_scalar_input]) -def test_omnisci_array_binary_math_scalar(omnisci, method, signature, args): - omnisci.reset() +def test_heavydb_array_binary_math_scalar(heavydb, method, signature, args): + heavydb.reset() s = f'def np_{method}(a, b): return array_api.{method}(a, b)' exec(s, globals()) - omnisci(signature)(eval('np_{}'.format(method))) + heavydb(signature)(eval('np_{}'.format(method))) t = tuple(args.split(',')) a, b = t[0], t[1] @@ -188,9 +188,9 @@ def test_omnisci_array_binary_math_scalar(omnisci, method, signature, args): query = f'select {column}, ' + \ f'np_{method}({a}, {b})' + \ - f' from {omnisci.table_name};' + f' from {heavydb.table_name};' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) row, out = list(result)[0] expected = getattr(np, method)(row, num(scalar)) @@ -256,19 +256,19 @@ def test_omnisci_array_binary_math_scalar(omnisci, method, signature, args): @pytest.mark.parametrize("method, signature, column", unary_fns, ids=[item[0] for item in unary_fns]) -def test_omnisci_array_unary_math_fns(omnisci, method, signature, column): - omnisci.reset() +def test_heavydb_array_unary_math_fns(heavydb, method, signature, column): + heavydb.reset() s = f'def np_{method}(a): return array_api.{method}(a)' exec(s, globals()) - omnisci(signature)(eval('np_{}'.format(method))) + heavydb(signature)(eval('np_{}'.format(method))) query = f'select {column}, ' + \ f'np_{method}({column})' + \ - f' from {omnisci.table_name};' + f' from {heavydb.table_name};' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) row, out = list(result)[0] if method == 'invert' and column == 'b': @@ -279,14 +279,14 @@ def test_omnisci_array_unary_math_fns(omnisci, method, signature, column): assert np.isclose(expected, out, equal_nan=True).all(), 'omni_' + method # noqa: E501 -def test_heaviside(omnisci): +def test_heaviside(heavydb): - @omnisci('double[](int64[], int64)') + @heavydb('double[](int64[], int64)') def heaviside(x1, x2): return array_api.heaviside(x1, x2) - query = f'select i8, heaviside(i8, 1) from {omnisci.table_name}' - _, result = omnisci.sql_execute(query) + query = f'select i8, heaviside(i8, 1) from {heavydb.table_name}' + _, result = heavydb.sql_execute(query) result = list(result) for inp, out in result: diff --git a/rbc/tests/test_omnisci_array_methods.py b/rbc/tests/heavyai/test_array_methods.py similarity index 80% rename from rbc/tests/test_omnisci_array_methods.py rename to rbc/tests/heavyai/test_array_methods.py index ff7b60d8..5d78bf0f 100644 --- a/rbc/tests/test_omnisci_array_methods.py +++ b/rbc/tests/heavyai/test_array_methods.py @@ -1,11 +1,11 @@ import pytest import numpy as np -from rbc.tests import omnisci_fixture -from rbc.omnisci_backend import Array +from rbc.tests import heavydb_fixture +from rbc.heavyai import Array -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @@ -13,8 +13,8 @@ @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): define(o) yield o @@ -48,15 +48,15 @@ def omnisci(): ] -def define(omnisci): +def define(heavydb): - @omnisci('double[](int64, double)') + @heavydb('double[](int64, double)') def ndarray_fill(size, v): a = Array(size, 'double') a.fill(v) return a - @omnisci('double(int64, double)') + @heavydb('double(int64, double)') def ndarray_max(size, v): a = Array(size, 'double') a.fill(v) @@ -71,29 +71,29 @@ def ndarray_max(size, v): exec(fn) fn = locals()[fn_name] if op == 'mean': - omnisci('float64(int32)')(fn) + heavydb('float64(int32)')(fn) else: - omnisci(f'{retty}(int32)')(fn) + heavydb(f'{retty}(int32)')(fn) - @omnisci('double(int64, double)') + @heavydb('double(int64, double)') def ndarray_mean(size, v): a = Array(size, 'double') a.fill(v) return a.mean() - @omnisci('double(int64, double)') + @heavydb('double(int64, double)') def ndarray_min(size, v): a = Array(size, 'double') a.fill(v) return a.min() - @omnisci('double(int64, double)') + @heavydb('double(int64, double)') def ndarray_sum(size, v): a = Array(size, 'double') a.fill(v) return a.sum() - @omnisci('double(int64, double)') + @heavydb('double(int64, double)') def ndarray_prod(size, v): a = Array(size, 'double') a.fill(v) @@ -102,11 +102,11 @@ def ndarray_prod(size, v): @pytest.mark.parametrize("method, args, expected", ndarray_methods, ids=[item[0] for item in ndarray_methods]) -def test_ndarray_methods(omnisci, method, args, expected): +def test_ndarray_methods(heavydb, method, args, expected): query_args = ', '.join(map(str, args)) query = f'SELECT ndarray_{method}({query_args})' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) out = list(result)[0] if method == 'fill': diff --git a/rbc/tests/test_omnisci_array_null.py b/rbc/tests/heavyai/test_array_null.py similarity index 73% rename from rbc/tests/test_omnisci_array_null.py rename to rbc/tests/heavyai/test_array_null.py index fabd2282..a5c4baeb 100644 --- a/rbc/tests/test_omnisci_array_null.py +++ b/rbc/tests/heavyai/test_array_null.py @@ -1,5 +1,5 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture ARRAY_NULL = 0 ARRAY_IDX_IS_NULL = 1 @@ -7,15 +7,15 @@ @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): +def heavydb(): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('int64(T[], int64)', T=['bool', 'int8', 'int16', 'int32', 'int64', 'float', 'double']) + @heavydb('int64(T[], int64)', T=['bool', 'int8', 'int16', 'int32', 'int64', 'float', 'double']) def array_null_check(x, index): if x.is_null(): # array row is null return ARRAY_NULL @@ -28,20 +28,20 @@ def array_null_check(x, index): @pytest.mark.parametrize('col', colnames) -def test_array_null(omnisci, col): +def test_array_null(heavydb, col): if col in ['i2', 'i8', 'f8']: - omnisci.require_version((5, 7, 0), - 'Requires omniscidb-internal PR 5465 [rbc PR 330]') + heavydb.require_version((5, 7, 0), + 'Requires heavydb-internal PR 5465 [rbc PR 330]') if col == 'b': - omnisci.require_version((5, 7, 0), - 'Requires omniscidb-internal PR 5492 [rbc issue 245]') + heavydb.require_version((5, 7, 0), + 'Requires heavydb-internal PR 5492 [rbc issue 245]') # Query null value - _, result = omnisci.sql_execute(f''' + _, result = heavydb.sql_execute(f''' SELECT array_null_check({col}, 0) FROM - {omnisci.table_name}arraynull + {heavydb.table_name}arraynull ''') result = list(result) diff --git a/rbc/tests/test_omnisci_array_operators.py b/rbc/tests/heavyai/test_array_operators.py similarity index 86% rename from rbc/tests/test_omnisci_array_operators.py rename to rbc/tests/heavyai/test_array_operators.py index 521487d9..b7592e3c 100644 --- a/rbc/tests/test_omnisci_array_operators.py +++ b/rbc/tests/heavyai/test_array_operators.py @@ -1,19 +1,19 @@ import pytest import numpy as np -from rbc.omnisci_backend import Array -from rbc.tests import omnisci_fixture +from rbc.heavyai import Array +from rbc.tests import heavydb_fixture from numba import types as nb_types import operator -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): define(o) yield o @@ -72,16 +72,16 @@ def omnisci(): ] -def define(omnisci): +def define(heavydb): - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_abs(size): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(-i) return abs(a) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_add(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -90,7 +90,7 @@ def operator_add(size): b[i] = nb_types.int32(size-i-1) return operator.add(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_and_bw(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -99,28 +99,28 @@ def operator_and_bw(size): b[i] = nb_types.int32(size-i-1) return operator.and_(a, b) - @omnisci('int64(int64, int64, int64)') + @heavydb('int64(int64, int64, int64)') def operator_countOf(size, fill_value, b): a = Array(size, 'int64') for i in range(size): a[i] = fill_value return operator.countOf(a, b) - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_eq(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a == v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_eq_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a == a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_floordiv(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -129,7 +129,7 @@ def operator_floordiv(size): b[i] = nb_types.int32(i+3) return operator.floordiv(a, b) - @omnisci('double[](int64)') + @heavydb('double[](int64)') def operator_floordiv2(size): a = Array(size, 'double') b = Array(size, 'double') @@ -138,35 +138,35 @@ def operator_floordiv2(size): b[i] = nb_types.double(i+3) return operator.floordiv(a, b) - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_ge(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a >= v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_ge_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a >= a - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_gt(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a > v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_gt_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a > a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_iadd(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -176,7 +176,7 @@ def operator_iadd(size): operator.iadd(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_iand(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -186,7 +186,7 @@ def operator_iand(size): operator.iand(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_ifloordiv(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -196,7 +196,7 @@ def operator_ifloordiv(size): operator.ifloordiv(a, b) return a - @omnisci('double[](int64)') + @heavydb('double[](int64)') def operator_ifloordiv2(size): a = Array(size, 'double') b = Array(size, 'double') @@ -206,7 +206,7 @@ def operator_ifloordiv2(size): operator.ifloordiv(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_ilshift(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -216,7 +216,7 @@ def operator_ilshift(size): operator.ilshift(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_imul(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -226,7 +226,7 @@ def operator_imul(size): operator.imul(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_ior(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -236,7 +236,7 @@ def operator_ior(size): operator.ior(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_isub(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -246,7 +246,7 @@ def operator_isub(size): operator.isub(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_ipow(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -256,7 +256,7 @@ def operator_ipow(size): operator.ipow(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_irshift(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -266,7 +266,7 @@ def operator_irshift(size): operator.irshift(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_itruediv(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -276,7 +276,7 @@ def operator_itruediv(size): operator.itruediv(a, b) return a - @omnisci('double[](int64)') + @heavydb('double[](int64)') def operator_itruediv2(size): a = Array(size, 'double') b = Array(size, 'double') @@ -286,7 +286,7 @@ def operator_itruediv2(size): operator.itruediv(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_imod(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -296,7 +296,7 @@ def operator_imod(size): operator.imod(a, b) return a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_ixor(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -306,26 +306,26 @@ def operator_ixor(size): operator.ixor(a, b) return a - @omnisci('int8(int64, int32)') + @heavydb('int8(int64, int32)') def operator_in(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return v in a - @omnisci('int8(int64, int32)') + @heavydb('int8(int64, int32)') def operator_is(size, v): a = Array(size, 'int32') a.fill(v) return a is a - @omnisci('int8(int64, int32)') + @heavydb('int8(int64, int32)') def operator_is_not(size, v): a = Array(size, 'int32') a.fill(v) return a is not a - @omnisci('int8(int64, int32)') + @heavydb('int8(int64, int32)') def operator_is_not2(size, v): a = Array(size, 'int32') a.fill(v) @@ -333,21 +333,21 @@ def operator_is_not2(size, v): b.fill(v) return a is not b - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_le(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a <= v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_le_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a <= a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_lshift(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -356,21 +356,21 @@ def operator_lshift(size): b[i] = nb_types.int32(size-i-1) return operator.lshift(a, b) - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_lt(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a < v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_lt_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a < a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_mul(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -379,7 +379,7 @@ def operator_mul(size): b[i] = nb_types.int32(size-i-1) return operator.mul(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_mod(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -388,35 +388,35 @@ def operator_mod(size): b[i] = nb_types.int32(7) return operator.mod(a, b) - @omnisci('int8[](int64, int32)') + @heavydb('int8[](int64, int32)') def operator_ne(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a != v - @omnisci('bool(int64, int32)') + @heavydb('bool(int64, int32)') def operator_ne_array(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return a != a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_neg(size): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return operator.neg(a) - @omnisci('int8(int64, int32)') + @heavydb('int8(int64, int32)') def operator_not_in(size, v): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(i) return v not in a - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_or_bw(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -425,14 +425,14 @@ def operator_or_bw(size): b[i] = nb_types.int32(size-i-1) return operator.or_(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_pos(size): a = Array(size, 'int32') for i in range(size): a[i] = nb_types.int32(-i) return operator.pos(a) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_pow(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -441,7 +441,7 @@ def operator_pow(size): b[i] = nb_types.int32(size-i) return operator.pow(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_rshift(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -450,7 +450,7 @@ def operator_rshift(size): b[i] = nb_types.int32(size-i-1) return operator.rshift(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_sub(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -459,7 +459,7 @@ def operator_sub(size): b[i] = nb_types.int32(size-i-1) return operator.sub(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_truediv(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -468,7 +468,7 @@ def operator_truediv(size): b[i] = nb_types.int32(i+3) return operator.truediv(a, b) - @omnisci('double[](int64)') + @heavydb('double[](int64)') def operator_truediv2(size): a = Array(size, 'double') b = Array(size, 'double') @@ -477,7 +477,7 @@ def operator_truediv2(size): b[i] = nb_types.double(i+3) return operator.truediv(a, b) - @omnisci('int32[](int64)') + @heavydb('int32[](int64)') def operator_xor(size): a = Array(size, 'int32') b = Array(size, 'int32') @@ -489,10 +489,10 @@ def operator_xor(size): @pytest.mark.parametrize("suffix, args, expected", operator_methods, ids=[item[0] for item in operator_methods]) -def test_array_operators(omnisci, suffix, args, expected): +def test_array_operators(heavydb, suffix, args, expected): query = 'select operator_{suffix}'.format(**locals()) + \ '(' + ', '.join(map(str, args)) + ')' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) out = list(result)[0] if suffix in ['in', 'not_in']: diff --git a/rbc/tests/test_omnisci_black_scholes.py b/rbc/tests/heavyai/test_black_scholes.py similarity index 77% rename from rbc/tests/test_omnisci_black_scholes.py rename to rbc/tests/heavyai/test_black_scholes.py index 8b6a0693..5d35061e 100644 --- a/rbc/tests/test_omnisci_black_scholes.py +++ b/rbc/tests/heavyai/test_black_scholes.py @@ -4,8 +4,8 @@ import numpy as np import math -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @@ -22,10 +22,10 @@ def read_csv(filename): @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) table_name = 'rbc_test_black_scholes' m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') @@ -71,14 +71,14 @@ def cnd_numba(d): return ret_val -def test_black_scholes_udf(omnisci): - omnisci.reset() +def test_black_scholes_udf(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('double(double, double, double, double, double)') + @heavydb('double(double, double, double, double, double)') def black_scholes_UDF(S, X, T, r, sigma): d1 = (np.log(S/X) + (r + 0.5 * sigma**2)*T) / (sigma * np.sqrt(T)) d2 = d1 - sigma * np.sqrt(T) @@ -94,23 +94,23 @@ def black_scholes_UDF(S, X, T, r, sigma): S = 100.0 r = 0.0 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select OPRICE, black_scholes_UDF(' f'cast({S} as DOUBLE), X, T, cast({r} as DOUBLE), sigma)' - f' from {omnisci.table_name}') + f' from {heavydb.table_name}') for _, (expected, out) in enumerate(result): assert math.isclose(expected, out, abs_tol=0.0001) -def test_black_scholes_udtf(omnisci): - omnisci.reset() +def test_black_scholes_udtf(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, Column, Column, Column, Column,' # noqa: E501 + @heavydb('int32(Column, Column, Column, Column, Column,' # noqa: E501 ' RowMultiplier, OutputColumn)') def black_scholes_udtf(S, X, T, r, sigma, m, out): input_row_count = len(X) @@ -129,17 +129,17 @@ def black_scholes_udtf(S, X, T, r, sigma, m, out): return m * input_row_count - _, result = omnisci.sql_execute( + _, result = heavydb.sql_execute( 'select * from table(black_scholes_udtf(' - ' cursor(select S from {omnisci.table_name}),' - ' cursor(select X from {omnisci.table_name}),' - ' cursor(select T from {omnisci.table_name}),' - ' cursor(select r from {omnisci.table_name}),' - ' cursor(select sigma from {omnisci.table_name}),' + ' cursor(select S from {heavydb.table_name}),' + ' cursor(select X from {heavydb.table_name}),' + ' cursor(select T from {heavydb.table_name}),' + ' cursor(select r from {heavydb.table_name}),' + ' cursor(select sigma from {heavydb.table_name}),' ' 1));' .format(**locals())) - _, oprice = omnisci.sql_execute(f'select OPRICE from {omnisci.table_name}') + _, oprice = heavydb.sql_execute(f'select OPRICE from {heavydb.table_name}') for r, o in zip(result, oprice): assert math.isclose(o[0], r[0], abs_tol=0.0001) diff --git a/rbc/tests/test_omnisci_caller.py b/rbc/tests/heavyai/test_caller.py similarity index 75% rename from rbc/tests/test_omnisci_caller.py rename to rbc/tests/heavyai/test_caller.py index 1a680da0..e6ee6e07 100644 --- a/rbc/tests/test_omnisci_caller.py +++ b/rbc/tests/heavyai/test_caller.py @@ -1,26 +1,26 @@ import numpy as np import pytest -from rbc.externals.omniscidb import set_output_row_size -from rbc.omnisci_backend import Bytes -from rbc.tests import omnisci_fixture, assert_equal +from rbc.externals.heavydb import set_output_row_size +from rbc.heavyai import Bytes +from rbc.tests import heavydb_fixture, assert_equal @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals(), minimal_version=(5, 8)): +def heavydb(): + for o in heavydb_fixture(globals(), minimal_version=(5, 8)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('T(T)', T=['int32', 'int64', 'float32', 'float64']) + @heavydb('T(T)', T=['int32', 'int64', 'float32', 'float64']) def myincr(x): return x + 1 T = ['int64', 'float64', 'int32'] - @omnisci('UDTF(int32 size, T x0, OutputColumn x)', T=T, devices=['cpu']) + @heavydb('UDTF(int32 size, T x0, OutputColumn x)', T=T, devices=['cpu']) def arange(size, x0, x): set_output_row_size(size) for i in range(size): @@ -29,7 +29,7 @@ def arange(size, x0, x): T = ['int64', 'float32'] - @omnisci('UDTF(Column x, T dx, OutputColumn y)', T=T, devices=['cpu']) + @heavydb('UDTF(Column x, T dx, OutputColumn y)', T=T, devices=['cpu']) def aincr(x, dx, y): size = len(x) set_output_row_size(size) @@ -37,7 +37,7 @@ def aincr(x, dx, y): y[i] = x[i] + dx return size - @omnisci('Bytes(Bytes)') + @heavydb('Bytes(Bytes)') def myupper(s): r = Bytes(len(s)) for i in range(len(s)): @@ -48,8 +48,8 @@ def myupper(s): return r -def test_udf_string_repr(omnisci): - myincr = omnisci.get_caller('myincr') +def test_udf_string_repr(heavydb): + myincr = heavydb.get_caller('myincr') assert_equal(repr(myincr), "RemoteDispatcher('myincr', ['T(T), T=int32|int64|float32|float64'])") @@ -65,8 +65,8 @@ def test_udf_string_repr(omnisci): "SELECT myincr(CAST(myincr(CAST(5 AS BIGINT)) AS BIGINT))") -def test_udtf_string_repr(omnisci): - arange = omnisci.get_caller('arange') +def test_udtf_string_repr(heavydb): + arange = heavydb.get_caller('arange') assert_equal(repr(arange), ("RemoteDispatcher('arange', ['UDTF(int32 size, T x0, OutputColumn x)," @@ -82,8 +82,8 @@ def test_udtf_string_repr(omnisci): "SELECT x FROM TABLE(arange(CAST(5 AS INT), CAST(0 AS BIGINT)))") -def test_remote_udf_evaluation(omnisci): - myincr = omnisci.get_caller('myincr') +def test_remote_udf_evaluation(heavydb): + myincr = heavydb.get_caller('myincr') assert_equal(str(myincr(3)), 'SELECT myincr(CAST(3 AS BIGINT))') assert_equal(myincr(3, hold=False), 4) @@ -94,26 +94,26 @@ def test_remote_udf_evaluation(omnisci): assert_equal(myincr(np.float32(3.5)).execute(), np.float32(4.5)) -def test_remote_int32_evaluation(omnisci): - myincr = omnisci.get_caller('myincr') - arange = omnisci.get_caller('arange') +def test_remote_int32_evaluation(heavydb): + myincr = heavydb.get_caller('myincr') + arange = heavydb.get_caller('arange') pytest.xfail('SELECT upcasts int32 to int64') assert_equal(myincr(np.int32(3)), np.int32(4)) assert_equal(arange(3, np.int32(1))['x'], np.arange(3, dtype=np.int32) + 1) -def test_remote_float64_evaluation(omnisci): - myincr = omnisci.get_caller('myincr') - arange = omnisci.get_caller('arange') +def test_remote_float64_evaluation(heavydb): + myincr = heavydb.get_caller('myincr') + arange = heavydb.get_caller('arange') pytest.xfail('SELECT downcasts float64 to float32') assert_equal(myincr(np.float64(3.5)), np.float64(4.5)) assert_equal(arange(3, np.float64(1))['x'], np.arange(3, dtype=np.float64) + 1) -def test_remote_bytes_evaluation(omnisci): - myupper = omnisci.get_caller('myupper') +def test_remote_bytes_evaluation(heavydb): + myupper = heavydb.get_caller('myupper') assert str(myupper) == "myupper['Bytes(Bytes)']" assert str(myupper("abc")) == "SELECT myupper('abc')" assert str(myupper("abc").execute()) == 'ABC' @@ -121,8 +121,8 @@ def test_remote_bytes_evaluation(omnisci): assert str(myupper(b"abc").execute()) == 'ABC' -def test_remote_composite_udf_evaluation(omnisci): - myincr = omnisci.get_caller('myincr') +def test_remote_composite_udf_evaluation(heavydb): + myincr = heavydb.get_caller('myincr') assert_equal(str(myincr(myincr(3))), 'SELECT myincr(CAST(myincr(CAST(3 AS BIGINT)) AS BIGINT))') @@ -131,8 +131,8 @@ def test_remote_composite_udf_evaluation(omnisci): assert_equal(myincr(myincr(3)).execute(), 5) -def test_remote_udtf_evaluation(omnisci): - arange = omnisci.get_caller('arange') +def test_remote_udtf_evaluation(heavydb): + arange = heavydb.get_caller('arange') assert_equal(str(arange(3, 1)), 'SELECT x FROM TABLE(arange(CAST(3 AS INT), CAST(1 AS BIGINT)))') @@ -144,10 +144,10 @@ def test_remote_udtf_evaluation(omnisci): assert_equal(arange(3, np.int64(1)).execute()['x'], np.arange(3, dtype=np.int64) + 1) -def test_remote_composite_udtf_evaluation(omnisci): - arange = omnisci.get_caller('arange') - aincr = omnisci.get_caller('aincr') - myincr = omnisci.get_caller('myincr') +def test_remote_composite_udtf_evaluation(heavydb): + arange = heavydb.get_caller('arange') + aincr = heavydb.get_caller('aincr') + myincr = heavydb.get_caller('myincr') r = aincr(arange(3, 1), 2) assert_equal(str(r), 'SELECT y FROM TABLE(aincr(CURSOR(SELECT x FROM' @@ -161,24 +161,24 @@ def test_remote_composite_udtf_evaluation(omnisci): assert_equal(r.execute()['x'], np.arange(3, dtype=np.int64) + 2 + 1) -def test_remote_composite_udtf_udf(omnisci): +def test_remote_composite_udtf_udf(heavydb): """ TableFunctionExecutionContext.cpp:277 Check failed: col_buf_ptrs.size() == exe_unit.input_exprs.size() (1 == 2) """ - myincr = omnisci.get_caller('myincr') - arange = omnisci.get_caller('arange') + myincr = heavydb.get_caller('myincr') + arange = heavydb.get_caller('arange') r = arange(3, myincr(2)) assert_equal(str(r), ('SELECT x FROM TABLE(arange(CAST(3 AS INT),' ' CAST(myincr(CAST(2 AS BIGINT)) AS BIGINT)))')) - pytest.xfail('udtf(udf) crashes omniscidb server') + pytest.xfail('udtf(udf) crashes heavydb server') assert_equal(r['x'], np.arange(3, dtype=np.int64) + 2 + 1) -def test_remote_udf_typeerror(omnisci): - myincr = omnisci.get_caller('myincr') +def test_remote_udf_typeerror(heavydb): + myincr = heavydb.get_caller('myincr') try: myincr("abc") except TypeError as msg: @@ -194,8 +194,8 @@ def test_remote_udf_typeerror(omnisci): assert 0 # expected TypeError -def test_remote_udtf_typeerror(omnisci): - arange = omnisci.get_caller('arange') +def test_remote_udtf_typeerror(heavydb): + arange = heavydb.get_caller('arange') try: arange(1.2, 0) except TypeError as msg: @@ -213,13 +213,13 @@ def test_remote_udtf_typeerror(omnisci): assert 0 # expected TypeError -def test_remote_udf_overload(omnisci): +def test_remote_udf_overload(heavydb): - @omnisci('int32(int32)') # noqa: F811 + @heavydb('int32(int32)') # noqa: F811 def incr_ol(x): # noqa: F811 return x + 1 - @omnisci('int32(int32, int32)') # noqa: F811 + @heavydb('int32(int32, int32)') # noqa: F811 def incr_ol(x, dx): # noqa: F811 return x + dx diff --git a/rbc/tests/test_omnisci_column_arguments.py b/rbc/tests/heavyai/test_column_arguments.py similarity index 79% rename from rbc/tests/test_omnisci_column_arguments.py rename to rbc/tests/heavyai/test_column_arguments.py index 5e3d0078..9aa51410 100644 --- a/rbc/tests/test_omnisci_column_arguments.py +++ b/rbc/tests/heavyai/test_column_arguments.py @@ -1,25 +1,25 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def text_rbc_copy_c_rowmul(x, m, y): for i in range(len(x)): # TODO: implement is_null support y[i] = x[i] return len(x) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def text_rbc_copy_cc_rowmul(x, x2, m, y, y2): for i in range(len(x)): @@ -27,7 +27,7 @@ def text_rbc_copy_cc_rowmul(x, x2, m, y, y2): y2[i] = x2[i] return len(x) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def text_rbc_copy_c_c_rowmul(x, x2, m, y, y2): for i in range(len(x)): @@ -35,7 +35,7 @@ def text_rbc_copy_c_c_rowmul(x, x2, m, y, y2): y2[i] = x2[i] return len(x) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn, OutputColumn)') def text_rbc_copy_c_cc_rowmul(x, x2, x3, m, y, y2, y3): for i in range(len(x)): @@ -44,7 +44,7 @@ def text_rbc_copy_c_cc_rowmul(x, x2, x3, m, y, y2, y3): y3[i] = x3[i] return len(x) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn, OutputColumn)') def text_rbc_copy_cc_c_rowmul(x, x2, x3, m, y, y2, y3): for i in range(len(x)): @@ -53,7 +53,7 @@ def text_rbc_copy_cc_c_rowmul(x, x2, x3, m, y, y2, y3): y3[i] = x3[i] return len(x) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn, OutputColumn,' ' OutputColumn)') def text_rbc_copy_cc_cc_rowmul(x, x2, x3, x4, m, y, y2, y3, y4): @@ -64,7 +64,7 @@ def text_rbc_copy_cc_cc_rowmul(x, x2, x3, x4, m, y, y2, y3, y4): y4[i] = x4[i] return len(x) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn, OutputColumn,' ' OutputColumn)') def text_rbc_copy_cccc_rowmul(x, x2, x3, x4, m, y, y2, y3, y4): @@ -81,12 +81,12 @@ def text_rbc_copy_cccc_rowmul(x, x2, x3, x4, m, y, y2, y3, y4): ['i8', 'cursor(i8,f8)', 'i8;f8', 'i4;cursor(f8,i8)', 'cursor(i4,i8);f8', 'cursor(i4,i8);cursor(f8,f4)', 'cursor(i4,i8,f8,f4)']) -def test_copy(omnisci, use_default, inputs): +def test_copy(heavydb, use_default, inputs): if use_default: - omnisci.require_version((5, 7), 'Requires omnisci-internal PR 5403') + heavydb.require_version((5, 7), 'Requires heavydb-internal PR 5403') groups = inputs.split(';') - table_names = [f'{omnisci.table_name}'] * len(groups) + table_names = [f'{heavydb.table_name}'] * len(groups) group_results = {} args = [] @@ -97,7 +97,7 @@ def test_copy(omnisci, use_default, inputs): colnames = colnames[7:-1] result = group_results.get(colnames) if result is None: - _, result = omnisci.sql_execute(f'select {colnames} from {table_name}') + _, result = heavydb.sql_execute(f'select {colnames} from {table_name}') result = group_results[colnames] = list(result) expected = [row1 + row2 for row1, row2 in zip(expected, result)] args.append(f'cursor(select {colnames} from {table_name})') @@ -108,21 +108,21 @@ def test_copy(omnisci, use_default, inputs): cc = '_'.join(cc) query = f'select * from table(text_rbc_copy_{cc}_rowmul({args}))' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) result = list(result) assert result == expected @pytest.mark.parametrize('kind', ['1', '11', '111', '211', '221', '212', '13', '32', '34', '242']) -def test_ct_binding_constant_sizer(omnisci, kind): - omnisci.require_version((5, 7), 'Requires omniscidb-internal PR 5274') +def test_ct_binding_constant_sizer(heavydb, kind): + heavydb.require_version((5, 7), 'Requires heavydb-internal PR 5274') colnames = ', '.join({'1': 'i4', '2': 'i8', '3': 'i4, i4, i4', '4': 'i8, i8, i8'}[n] for n in kind) query = (f'select * from table(ct_binding_udtf_constant(cursor(' - f'select {colnames} from {omnisci.table_name})))') - _, result = omnisci.sql_execute(query) + f'select {colnames} from {heavydb.table_name})))') + _, result = heavydb.sql_execute(query) result = list(result) assert result == [(int(kind),)] @@ -132,11 +132,11 @@ def test_ct_binding_constant_sizer(omnisci, kind): @pytest.mark.parametrize('kind', ['19', '119', '1119', '2119', '2219', '2129', '139', '329', '349', '2429', '91', '196', '396', '369', '169']) -def test_ct_binding_row_multiplier(omnisci, use_default, kind): +def test_ct_binding_row_multiplier(heavydb, use_default, kind): if use_default: - omnisci.require_version((5, 7), 'Requires omnisci-internal PR 5403') + heavydb.require_version((5, 7), 'Requires heavydb-internal PR 5403') - if omnisci.version < (5, 7): + if heavydb.version < (5, 7): suffix = {'91': '2', '369': '2', '169': '3'}.get(kind, '') codes = {'1': 'i4', '2': 'i8', '3': 'i4, i4, i4', '4': 'i8, i8, i8', '9': '1', '6': 'cast(123 as int)'} @@ -156,8 +156,8 @@ def test_ct_binding_row_multiplier(omnisci, use_default, kind): last = ', '.join([''] + last) cursor = ', '.join(cursor) query = (f'select * from table(ct_binding_udtf{suffix}({first}cursor(' - f'select {cursor} from {omnisci.table_name}){last}))') - _, result = omnisci.sql_execute(query) + f'select {cursor} from {heavydb.table_name}){last}))') + _, result = heavydb.sql_execute(query) result = list(result) assert result == [(1000 + int(kind),)], (result, query) @@ -189,8 +189,8 @@ def test_ct_binding_row_multiplier(omnisci, use_default, kind): last = ', '.join([''] + last) cursor = ', '.join(cursor) query = (f'select * from table(ct_binding_udtf{suffix}({first}cursor(' - f'select {cursor} from {omnisci.table_name}){last}))') - _, result = omnisci.sql_execute(query) + f'select {cursor} from {heavydb.table_name}){last}))') + _, result = heavydb.sql_execute(query) result = list(result) assert result == [(1000 + int(kind) + 1 + 10*input_val,)], (result, query) diff --git a/rbc/tests/test_omnisci_column_basic.py b/rbc/tests/heavyai/test_column_basic.py similarity index 65% rename from rbc/tests/test_omnisci_column_basic.py rename to rbc/tests/heavyai/test_column_basic.py index e24df0e2..d502a095 100644 --- a/rbc/tests/test_omnisci_column_basic.py +++ b/rbc/tests/heavyai/test_column_basic.py @@ -5,16 +5,16 @@ import numpy as np -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) table_name = os.path.splitext(os.path.basename(__file__))[0] m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') @@ -42,14 +42,14 @@ def omnisci(): m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') -def test_sizer_row_multiplier_orig(omnisci): - omnisci.reset() +def test_sizer_row_multiplier_orig(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def my_row_copier_mul(x, m, y): input_row_count = len(x) for i in range(input_row_count): @@ -58,22 +58,22 @@ def my_row_copier_mul(x, m, y): y[j] = x[i] * 2 return m * input_row_count - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_mul(cursor(select f8 ' - f'from {omnisci.table_name}), 2));') + f'from {heavydb.table_name}), 2));') for i, r in enumerate(result): assert r == (float((i % 5) * 2),) -def test_sizer_row_multiplier_param1(omnisci): - omnisci.reset() +def test_sizer_row_multiplier_param1(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, double, int32, RowMultiplier,' + @heavydb('int32(Column, double, int32, RowMultiplier,' 'OutputColumn)') def my_row_copier_mul_param1(x, alpha, beta, m, y): input_row_count = len(x) @@ -85,9 +85,9 @@ def my_row_copier_mul_param1(x, alpha, beta, m, y): alpha = 3 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_mul_param1(' - f'cursor(select f8 from {omnisci.table_name}),' + f'cursor(select f8 from {heavydb.table_name}),' f'cast({alpha} as double),' 'cast(4 as int), 2));') @@ -95,14 +95,14 @@ def my_row_copier_mul_param1(x, alpha, beta, m, y): assert r == ((i % 5) * alpha + 4,) -def test_sizer_row_multiplier_param2(omnisci): - omnisci.reset() +def test_sizer_row_multiplier_param2(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(double, Column, int32, RowMultiplier,' + @heavydb('int32(double, Column, int32, RowMultiplier,' 'OutputColumn)') def my_row_copier_mul_param2(alpha, x, beta, m, y): input_row_count = len(x) @@ -114,24 +114,24 @@ def my_row_copier_mul_param2(alpha, x, beta, m, y): alpha = 3 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_mul_param2(' f'cast({alpha} as double),' - f'cursor(select f8 from {omnisci.table_name}),' + f'cursor(select f8 from {heavydb.table_name}),' 'cast(4 as int), 2));') for i, r in enumerate(result): assert r == ((i % 5) * alpha + 4,) -def test_sizer_constant_parameter(omnisci): - omnisci.reset() +def test_sizer_constant_parameter(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, ConstantParameter, OutputColumn)') + @heavydb('int32(Column, ConstantParameter, OutputColumn)') def my_row_copier_cp(x, m, y): n = len(x) for i in range(m): @@ -139,54 +139,54 @@ def my_row_copier_cp(x, m, y): y[i] = x[j] * 2 return m - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_cp(cursor(select f8 ' - f'from {omnisci.table_name}), 3));') + f'from {heavydb.table_name}), 3));') result = list(result) assert len(result) == 3 for i, r in enumerate(result): assert r == (i * 2,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_cp(cursor(select f8 ' - f'from {omnisci.table_name}), 8));') + f'from {heavydb.table_name}), 8));') result = list(result) assert len(result) == 8 for i, r in enumerate(result): assert r == ((i % 5) * 2,) -def test_sizer_return_size(omnisci): - omnisci.reset() +def test_sizer_return_size(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, OutputColumn)') + @heavydb('int32(Column, OutputColumn)') def my_row_copier_c(x, y): for i in range(13): j = i % len(x) y[i] = x[j] * 2 return 13 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(my_row_copier_c(cursor(select f8 ' - f'from {omnisci.table_name})));') + f'from {heavydb.table_name})));') result = list(result) assert len(result) == 13 for i, r in enumerate(result): assert r == ((i % 5) * 2,), repr((i, r)) -def test_rowmul_add_columns(omnisci): - omnisci.reset() +def test_rowmul_add_columns(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, Column, double,' + @heavydb('int32(Column, Column, double,' ' RowMultiplier, OutputColumn)') def add_columns(x, y, alpha, m, r): input_row_count = len(x) @@ -198,24 +198,24 @@ def add_columns(x, y, alpha, m, r): alpha = 2.5 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(add_columns(' - f'cursor(select f8 from {omnisci.table_name}),' - f' cursor(select d from {omnisci.table_name}),' + f'cursor(select f8 from {heavydb.table_name}),' + f' cursor(select d from {heavydb.table_name}),' f' cast({alpha} as double), 1));') for i, r in enumerate(result): assert r == (i + alpha * (i + 1.5),) -def test_rowmul_return_mixed_columns(omnisci): - omnisci.reset() +def test_rowmul_return_mixed_columns(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - @omnisci('int32(Column, RowMultiplier,' + @heavydb('int32(Column, RowMultiplier,' ' OutputColumn, OutputColumn)') def ret_mixed_columns(x, m, y, z): input_row_count = len(x) @@ -226,25 +226,25 @@ def ret_mixed_columns(x, m, y, z): z[j] = np.float32(3 * x[i]) return m * input_row_count - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(ret_mixed_columns(' - f'cursor(select f8 from {omnisci.table_name}), 1));') + f'cursor(select f8 from {heavydb.table_name}), 1));') for i, r in enumerate(result): assert r[0] == float(2 * i) assert r[1] == float(3 * i) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(ret_mixed_columns(' - f'cursor(select f8 from {omnisci.table_name}), 2));') + f'cursor(select f8 from {heavydb.table_name}), 2));') for i, r in enumerate(result): assert r[0] == float(2 * (i % 5)) assert r[1] == float(3 * (i % 5)) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select out1, out0 from table(ret_mixed_columns(' - f'cursor(select f8 from {omnisci.table_name}), 1));') + f'cursor(select f8 from {heavydb.table_name}), 1));') for i, r in enumerate(result): assert r[1] == float(2 * i) @@ -254,15 +254,15 @@ def ret_mixed_columns(x, m, y, z): @pytest.mark.parametrize("max_n", [-1, 3]) @pytest.mark.parametrize("sizer", [1, 2]) @pytest.mark.parametrize("num_columns", [1, 2, 3, 4]) -def test_rowmul_return_multi_columns(omnisci, num_columns, sizer, max_n): - omnisci.reset() +def test_rowmul_return_multi_columns(heavydb, num_columns, sizer, max_n): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() if num_columns == 1: - @omnisci('int32(int32, Column, RowMultiplier,' + @heavydb('int32(int32, Column, RowMultiplier,' ' OutputColumn)') def ret_1_columns(n, x1, m, y1): input_row_count = len(x1) @@ -275,7 +275,7 @@ def ret_1_columns(n, x1, m, y1): return n if num_columns == 2: - @omnisci('int32(int32, Column, RowMultiplier,' + @heavydb('int32(int32, Column, RowMultiplier,' ' OutputColumn, OutputColumn)') def ret_2_columns(n, x1, m, y1, y2): input_row_count = len(x1) @@ -289,7 +289,7 @@ def ret_2_columns(n, x1, m, y1, y2): return n if num_columns == 3: - @omnisci('int32(int32, Column, RowMultiplier,' + @heavydb('int32(int32, Column, RowMultiplier,' ' OutputColumn, OutputColumn,' ' OutputColumn)') def ret_3_columns(n, x1, m, y1, y2, y3): @@ -305,7 +305,7 @@ def ret_3_columns(n, x1, m, y1, y2, y3): return n if num_columns == 4: - @omnisci('int32(int32, Column, RowMultiplier,' + @heavydb('int32(int32, Column, RowMultiplier,' ' OutputColumn, OutputColumn,' ' OutputColumn, OutputColumn)') def ret_4_columns(n, x1, m, y1, y2, y3, y4): @@ -321,9 +321,9 @@ def ret_4_columns(n, x1, m, y1, y2, y3, y4): return m * input_row_count return n - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select * from table(ret_{num_columns}_columns({max_n}, ' - f'cursor(select f8 from {omnisci.table_name}), {sizer}));') + f'cursor(select f8 from {heavydb.table_name}), {sizer}));') result = list(result) @@ -337,9 +337,9 @@ def ret_4_columns(n, x1, m, y1, y2, y3, y4): @pytest.mark.parametrize("variant", [1, 2, 3]) -def test_issue173(omnisci, variant): - omnisci.reset() - omnisci.register() +def test_issue173(heavydb, variant): + heavydb.reset() + heavydb.register() def mask_zero(x, b, m, y): for i in range(len(x)): @@ -349,34 +349,34 @@ def mask_zero(x, b, m, y): y[i] = x[i] return len(x) - descr, result = omnisci.sql_execute( - f'select f8, b from {omnisci.table_name}') + descr, result = heavydb.sql_execute( + f'select f8, b from {heavydb.table_name}') f8, b = zip(*list(result)) # remove after resolving rbc issue 175: mask_zero.__name__ += f'_{variant}' if variant == 1: - omnisci('int32(Column, Column, RowMultiplier,' + heavydb('int32(Column, Column, RowMultiplier,' ' OutputColumn)')(mask_zero) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select * from table({mask_zero.__name__}(' - f'cursor(select f8 from {omnisci.table_name}),' - f'cursor(select b from {omnisci.table_name}), 1))') + f'cursor(select f8 from {heavydb.table_name}),' + f'cursor(select b from {heavydb.table_name}), 1))') elif variant == 2: - omnisci('int32(Cursor, Column>, RowMultiplier,' + heavydb('int32(Cursor, Column>, RowMultiplier,' ' OutputColumn)')(mask_zero) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select * from table({mask_zero.__name__}(' - f'cursor(select f8, b from {omnisci.table_name}), 1))') + f'cursor(select f8, b from {heavydb.table_name}), 1))') elif variant == 3: - omnisci('int32(Cursor, RowMultiplier,' + heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn)')(mask_zero) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select * from table({mask_zero.__name__}(' - f'cursor(select f8, b from {omnisci.table_name}), 1))') + f'cursor(select f8, b from {heavydb.table_name}), 1))') else: raise ValueError(variant) @@ -386,104 +386,104 @@ def mask_zero(x, b, m, y): assert result[i][0] == (0.0 if b[i] else f8[i]) -def test_redefine(omnisci): - omnisci.reset() +def test_redefine(heavydb): + heavydb.reset() # register an empty set of UDFs in order to avoid unregistering # UDFs created directly from LLVM IR strings when executing SQL # queries: - omnisci.register() + heavydb.register() - descr, result = omnisci.sql_execute( - f'select f8, i4 from {omnisci.table_name}') + descr, result = heavydb.sql_execute( + f'select f8, i4 from {heavydb.table_name}') f8, i4 = zip(*result) - @omnisci('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 def redefined_udtf(x, m, y): # noqa: E501, F811 for i in range(len(x)): y[i] = x[i] + 1 return len(x) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(redefined_udtf(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') result = list(result) for i in range(len(result)): assert result[i][0] == f8[i] + 1 # redefine implementation - @omnisci('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 def redefined_udtf(x, m, y): # noqa: E501, F811 for i in range(len(x)): y[i] = x[i] + 2 return len(x) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(redefined_udtf(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') result = list(result) for i in range(len(result)): assert result[i][0] == f8[i] + 2 # overload with another type - @omnisci('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 def redefined_udtf(x, m, y): # noqa: E501, F811 for i in range(len(x)): y[i] = x[i] + 3 return len(x) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(redefined_udtf(cursor(' - f'select i4 from {omnisci.table_name}), 1))') + f'select i4 from {heavydb.table_name}), 1))') result = list(result) for i in range(len(result)): assert result[i][0] == i4[i] + 3 # check overload - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(redefined_udtf(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') result = list(result) for i in range(len(result)): assert result[i][0] == f8[i] + 2 @pytest.mark.parametrize("step", [1, 2, 3]) -def test_overload_nonuniform(omnisci, step): +def test_overload_nonuniform(heavydb, step): pytest.xfail('Test failing due to the introduction of default sizer. See PR 313') - omnisci.reset() - omnisci.register() + heavydb.reset() + heavydb.register() if step > 0: - @omnisci('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 def overloaded_udtf(x, m, y): # noqa: E501, F811 y[0] = 64 return 1 if step > 1: - @omnisci('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Column, RowMultiplier, OutputColumn)') # noqa: E501, F811 def overloaded_udtf(x, m, y): # noqa: E501, F811 y[0] = 32 return 1 if step > 2: - @omnisci('int32(Cursor, Column>, RowMultiplier, OutputColumn)') # noqa: E501, F811 + @heavydb('int32(Cursor, Column>, RowMultiplier, OutputColumn)') # noqa: E501, F811 def overloaded_udtf(x1, x2, m, y): # noqa: E501, F811 y[0] = 132 return 1 sql_query = ('select * from table(overloaded_udtf(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') if step > 0: - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result[0][0] == 64 sql_query = ('select * from table(overloaded_udtf(cursor(' - f'select f4 from {omnisci.table_name}), 1))') + f'select f4 from {heavydb.table_name}), 1))') if step > 1: - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result[0][0] == 32 else: @@ -491,12 +491,12 @@ def overloaded_udtf(x1, x2, m, y): # noqa: E501, F811 Exception, match=(r".*(Function overloaded_udtf\(COLUMN, INTEGER\) not supported" r"|Could not bind overloaded_udtf\(COLUMN, INTEGER\))")): - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) sql_query = ('select * from table(overloaded_udtf(cursor(' - f'select f4, f4 from {omnisci.table_name}), 1))') + f'select f4, f4 from {heavydb.table_name}), 1))') if step > 2: - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result[0][0] == 132 else: @@ -505,14 +505,14 @@ def overloaded_udtf(x1, x2, m, y): # noqa: E501, F811 match=(r".*(Function overloaded_udtf\(COLUMN," r" COLUMN, INTEGER\) not supported|Could not bind" r" overloaded_udtf\(COLUMN, COLUMN, INTEGER\))")): - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) -def test_overload_uniform(omnisci): - omnisci.reset() - omnisci.register() +def test_overload_uniform(heavydb): + heavydb.reset() + heavydb.register() - @omnisci('int32(Column, RowMultiplier, OutputColumn)', + @heavydb('int32(Column, RowMultiplier, OutputColumn)', T=['double', 'float', 'int64', 'int32', 'int16', 'int8']) def mycopy(x, m, y): # noqa: E501, F811 for i in range(len(x)): @@ -520,68 +520,68 @@ def mycopy(x, m, y): # noqa: E501, F811 return len(x) for colname in ['f8', 'f4', 'i8', 'i4', 'i2', 'i1', 'b']: - sql_query = (f'select {colname} from {omnisci.table_name}') - descr, result = omnisci.sql_execute(sql_query) + sql_query = (f'select {colname} from {heavydb.table_name}') + descr, result = heavydb.sql_execute(sql_query) expected = list(result) sql_query = ('select * from table(mycopy(cursor(' - f'select {colname} from {omnisci.table_name}), 1))') - descr, result = omnisci.sql_execute(sql_query) + f'select {colname} from {heavydb.table_name}), 1))') + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == expected -omnisci_aggregators = [ +heavydb_aggregators = [ 'avg', 'min', 'max', 'sum', 'count', 'approx_count_distinct', 'sample', 'single_value', 'stddev', 'stddev_pop', 'stddev_samp', 'variance', 'var_pop', 'var_samp', ] -omnisci_aggregators2 = ['correlation', 'corr', 'covar_pop', 'covar_samp'] +heavydb_aggregators2 = ['correlation', 'corr', 'covar_pop', 'covar_samp'] # TODO: round_to_digit, mod, truncate -omnisci_functions = ['sin', 'cos', 'tan', 'asin', 'acos', 'atan', 'abs', 'ceil', 'degrees', +heavydb_functions = ['sin', 'cos', 'tan', 'asin', 'acos', 'atan', 'abs', 'ceil', 'degrees', 'exp', 'floor', 'ln', 'log', 'log10', 'radians', 'round', 'sign', 'sqrt'] -omnisci_functions2 = ['atan2', 'power'] -omnisci_unary_operations = ['+', '-'] -omnisci_binary_operations = ['+', '-', '*', '/'] +heavydb_functions2 = ['atan2', 'power'] +heavydb_unary_operations = ['+', '-'] +heavydb_binary_operations = ['+', '-', '*', '/'] @pytest.mark.parametrize("prop", ['', 'groupby']) -@pytest.mark.parametrize("oper", omnisci_aggregators + omnisci_aggregators2) -def test_column_aggregate(omnisci, prop, oper): - omnisci.reset() - omnisci.register() +@pytest.mark.parametrize("oper", heavydb_aggregators + heavydb_aggregators2) +def test_column_aggregate(heavydb, prop, oper): + heavydb.reset() + heavydb.register() if prop == 'groupby': - pytest.skip('omniscidb server crashes') + pytest.skip('heavydb server crashes') if oper == 'single_value': if prop: return - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def test_rbc_last(x, m, y): y[0] = x[len(x)-1] return 1 - sql_query = (f'select f8 from {omnisci.table_name}') - descr, result = omnisci.sql_execute(sql_query) + sql_query = (f'select f8 from {heavydb.table_name}') + descr, result = heavydb.sql_execute(sql_query) result = list(result) expected_result = result[-1:] sql_query = (f'select {oper}(out0) from table(test_rbc_last(cursor(' - f'select f8 from {omnisci.table_name}), 1))') - descr, result = omnisci.sql_execute(sql_query) + f'select f8 from {heavydb.table_name}), 1))') + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == expected_result return - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def test_rbc_mycopy(x, m, y): for i in range(len(x)): y[i] = x[i] return len(x) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def test_rbc_mycopy2(x1, x2, m, y1, y2): for i in range(len(x1)): @@ -589,7 +589,7 @@ def test_rbc_mycopy2(x1, x2, m, y1, y2): y2[i] = x2[i] return len(x1) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def test_rbc_mycopy2b(x1, x2, m, y1, y2): for i in range(len(x1)): @@ -597,7 +597,7 @@ def test_rbc_mycopy2b(x1, x2, m, y1, y2): y2[i] = x2[i] return len(x1) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' 'OutputColumn, OutputColumn, OutputColumn)') def test_rbc_mycopy3(x1, x2, x3, m, y1, y2, y3): for i in range(len(x1)): @@ -612,51 +612,51 @@ def test_rbc_mycopy3(x1, x2, x3, m, y1, y2, y3): mycopy = 'test_rbc_mycopy' - if oper in omnisci_aggregators2: + if oper in heavydb_aggregators2: if prop == 'groupby': sql_query_expected = ( - f'select {oper}(f8, d) from {omnisci.table_name} group by b') + f'select {oper}(f8, d) from {heavydb.table_name} group by b') sql_query = ( f'select {oper}(out0, out1) from table({mycopy}3(cursor(' - f'select f8, d, b from {omnisci.table_name}), 1)) group by out2') + f'select f8, d, b from {heavydb.table_name}), 1)) group by out2') else: - sql_query_expected = (f'select {oper}(f8, d) from {omnisci.table_name}') + sql_query_expected = (f'select {oper}(f8, d) from {heavydb.table_name}') sql_query = ( f'select {oper}(out0, out1) from table({mycopy}2(cursor(' - f'select f8, d from {omnisci.table_name}), 1))') + f'select f8, d from {heavydb.table_name}), 1))') else: if prop == 'groupby': sql_query_expected = ( - f'select {oper}(f8{extra_args}) from {omnisci.table_name} group by b') + f'select {oper}(f8{extra_args}) from {heavydb.table_name} group by b') sql_query = ( f'select {oper}(out0) from table({mycopy}2b(cursor(' - f'select f8, b from {omnisci.table_name}), 1)) group by out1') + f'select f8, b from {heavydb.table_name}), 1)) group by out1') else: - sql_query_expected = (f'select {oper}(f8{extra_args}) from {omnisci.table_name}') + sql_query_expected = (f'select {oper}(f8{extra_args}) from {heavydb.table_name}') sql_query = ( f'select {oper}(out0) from table({mycopy}(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected -@pytest.mark.parametrize("oper", omnisci_functions + omnisci_functions2) -def test_column_function(omnisci, oper): - omnisci.reset() - omnisci.register() +@pytest.mark.parametrize("oper", heavydb_functions + heavydb_functions2) +def test_column_function(heavydb, oper): + heavydb.reset() + heavydb.register() - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def test_rbc_mycopy(x, m, y): for i in range(len(x)): y[i] = x[i] return len(x) - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def test_rbc_mycopy2(x1, x2, m, y1, y2): for i in range(len(x1)): @@ -666,21 +666,21 @@ def test_rbc_mycopy2(x1, x2, m, y1, y2): mycopy = 'test_rbc_mycopy' - if oper in omnisci_functions2: - sql_query_expected = (f'select {oper}(f8, d) from {omnisci.table_name}') + if oper in heavydb_functions2: + sql_query_expected = (f'select {oper}(f8, d) from {heavydb.table_name}') sql_query = ( f'select {oper}(out0, out1) from table({mycopy}2(cursor(' - f'select f8, d from {omnisci.table_name}), 1))') + f'select f8, d from {heavydb.table_name}), 1))') else: - sql_query_expected = (f'select {oper}(f8) from {omnisci.table_name}') + sql_query_expected = (f'select {oper}(f8) from {heavydb.table_name}') sql_query = ( f'select {oper}(out0) from table({mycopy}(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) if oper in ['asin', 'acos']: @@ -690,12 +690,12 @@ def test_rbc_mycopy2(x1, x2, m, y1, y2): assert result == result_expected -@pytest.mark.parametrize("oper", omnisci_binary_operations) -def test_column_binary_operation(omnisci, oper): - omnisci.reset() - omnisci.register() +@pytest.mark.parametrize("oper", heavydb_binary_operations) +def test_column_binary_operation(heavydb, oper): + heavydb.reset() + heavydb.register() - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)') def test_rbc_mycopy2(x1, x2, m, y1, y2): for i in range(len(x1)): @@ -705,26 +705,26 @@ def test_rbc_mycopy2(x1, x2, m, y1, y2): mycopy = 'test_rbc_mycopy' - sql_query_expected = (f'select f8 {oper} d from {omnisci.table_name}') + sql_query_expected = (f'select f8 {oper} d from {heavydb.table_name}') sql_query = ( f'select out0 {oper} out1 from table({mycopy}2(cursor(' - f'select f8, d from {omnisci.table_name}), 1))') + f'select f8, d from {heavydb.table_name}), 1))') - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected -@pytest.mark.parametrize("oper", omnisci_unary_operations) -def test_column_unary_operation(omnisci, oper): - omnisci.reset() - omnisci.register() +@pytest.mark.parametrize("oper", heavydb_unary_operations) +def test_column_unary_operation(heavydb, oper): + heavydb.reset() + heavydb.register() - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn)') def test_rbc_mycopy(x1, m, y1): for i in range(len(x1)): @@ -733,25 +733,25 @@ def test_rbc_mycopy(x1, m, y1): mycopy = 'test_rbc_mycopy' - sql_query_expected = (f'select {oper} f8 from {omnisci.table_name}') + sql_query_expected = (f'select {oper} f8 from {heavydb.table_name}') sql_query = ( f'select {oper} out0 from table({mycopy}(cursor(' - f'select f8 from {omnisci.table_name}), 1))') + f'select f8 from {heavydb.table_name}), 1))') - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected -def test_create_as(omnisci): - omnisci.reset() - omnisci.register() +def test_create_as(heavydb): + heavydb.reset() + heavydb.register() - @omnisci('int32(Cursor, RowMultiplier,' + @heavydb('int32(Cursor, RowMultiplier,' ' OutputColumn)') def test_rbc_mycopy(x1, m, y1): for i in range(len(x1)): @@ -760,48 +760,48 @@ def test_rbc_mycopy(x1, m, y1): mycopy = 'test_rbc_mycopy' - sql_query_expected = (f'select f8 from {omnisci.table_name}') + sql_query_expected = (f'select f8 from {heavydb.table_name}') sql_queries = [ - f'DROP TABLE IF EXISTS {omnisci.table_name}_f8', - (f'CREATE TABLE {omnisci.table_name}_f8 AS SELECT out0 FROM TABLE({mycopy}(CURSOR(' - f'SELECT f8 FROM {omnisci.table_name}), 1))'), - f'SELECT * FROM {omnisci.table_name}_f8'] + f'DROP TABLE IF EXISTS {heavydb.table_name}_f8', + (f'CREATE TABLE {heavydb.table_name}_f8 AS SELECT out0 FROM TABLE({mycopy}(CURSOR(' + f'SELECT f8 FROM {heavydb.table_name}), 1))'), + f'SELECT * FROM {heavydb.table_name}_f8'] - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) for sql_query in sql_queries: - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected @pytest.fixture(scope='function') -def create_columns(omnisci): +def create_columns(heavydb): # delete tables - omnisci.sql_execute('DROP TABLE IF EXISTS datatable;') - omnisci.sql_execute('DROP TABLE IF EXISTS kerneltable;') + heavydb.sql_execute('DROP TABLE IF EXISTS datatable;') + heavydb.sql_execute('DROP TABLE IF EXISTS kerneltable;') # create tables - omnisci.sql_execute('CREATE TABLE IF NOT EXISTS datatable (x DOUBLE);') - omnisci.sql_execute('CREATE TABLE IF NOT EXISTS kerneltable (kernel DOUBLE);') + heavydb.sql_execute('CREATE TABLE IF NOT EXISTS datatable (x DOUBLE);') + heavydb.sql_execute('CREATE TABLE IF NOT EXISTS kerneltable (kernel DOUBLE);') # add data - omnisci.load_table_columnar('datatable', **{'x': [1.0, 2.0, 3.0, 4.0, 5.0]}) - omnisci.load_table_columnar('kerneltable', **{'kernel': [10.0, 20.0, 30.0]}) - yield omnisci + heavydb.load_table_columnar('datatable', **{'x': [1.0, 2.0, 3.0, 4.0, 5.0]}) + heavydb.load_table_columnar('kerneltable', **{'kernel': [10.0, 20.0, 30.0]}) + yield heavydb # delete tables - omnisci.sql_execute('DROP TABLE IF EXISTS datatable;') - omnisci.sql_execute('DROP TABLE IF EXISTS kerneltable;') + heavydb.sql_execute('DROP TABLE IF EXISTS datatable;') + heavydb.sql_execute('DROP TABLE IF EXISTS kerneltable;') @pytest.mark.usefixtures('create_columns') -def test_column_different_sizes(omnisci): +def test_column_different_sizes(heavydb): - if omnisci.has_cuda and omnisci.version[:2] == (5, 10): + if heavydb.has_cuda and heavydb.version[:2] == (5, 10): pytest.xfail('Different result') - @omnisci('int32(Column, Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, Column, RowMultiplier, OutputColumn)') def convolve(x, kernel, m, y): for i in range(len(y)): y[i] = 0.0 @@ -813,7 +813,7 @@ def convolve(x, kernel, m, y): # output has the same size as @x return len(x) - _, result = omnisci.sql_execute( + _, result = heavydb.sql_execute( 'select * from table(' 'convolve(cursor(select x from datatable),' 'cursor(select kernel from kerneltable), 1))') @@ -822,11 +822,11 @@ def convolve(x, kernel, m, y): assert list(result) == expected -def test_column_dtype(omnisci): +def test_column_dtype(heavydb): from numba import types - table = omnisci.table_name + table = heavydb.table_name - @omnisci('int32(Column, RowMultiplier, OutputColumn)', + @heavydb('int32(Column, RowMultiplier, OutputColumn)', T=['int32', 'int64', 'float', 'double']) def col_dtype_fn(x, m, y): sz = len(x) @@ -851,14 +851,14 @@ def col_dtype_fn(x, m, y): ) for col, r in inpts: query = f'select * from table(col_dtype_fn(cursor(select {col} from {table}), 1))' - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(r,)] * 5 -def test_column_enumerate(omnisci): - from rbc.externals.omniscidb import set_output_row_size +def test_column_enumerate(heavydb): + from rbc.externals.heavydb import set_output_row_size - @omnisci('int32(Column, OutputColumn)') + @heavydb('int32(Column, OutputColumn)') def col_enumerate(x, y): sz = len(x) set_output_row_size(sz) @@ -866,9 +866,9 @@ def col_enumerate(x, y): y[i] = e return sz - _, result = omnisci.sql_execute( - f'select * from table(col_enumerate(cursor(select i4 from {omnisci.table_name})))') - _, expected_result = omnisci.sql_execute( - f'select rowid, i4 from {omnisci.table_name} order by rowid;') + _, result = heavydb.sql_execute( + f'select * from table(col_enumerate(cursor(select i4 from {heavydb.table_name})))') + _, expected_result = heavydb.sql_execute( + f'select rowid, i4 from {heavydb.table_name} order by rowid;') for (r,), (_, e) in zip(list(result), list(expected_result)): assert r == e diff --git a/rbc/tests/test_omnisci_column_default.py b/rbc/tests/heavyai/test_column_default.py similarity index 82% rename from rbc/tests/test_omnisci_column_default.py rename to rbc/tests/heavyai/test_column_default.py index e01d96d5..463e8da1 100644 --- a/rbc/tests/test_omnisci_column_default.py +++ b/rbc/tests/heavyai/test_column_default.py @@ -1,28 +1,28 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope="module") -def omnisci(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): +def heavydb(): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) yield o -def define(omnisci): - @omnisci("int32(Column, RowMultiplier, OutputColumn)") +def define(heavydb): + @heavydb("int32(Column, RowMultiplier, OutputColumn)") def rbc_udtf_default_sizer_1(c, sizer, o): o[0] = 3 return 1 -def test_python_fn_with_default_sizer(omnisci): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5403") +def test_python_fn_with_default_sizer(heavydb): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5403") - table = omnisci.table_name + table = heavydb.table_name fn = "rbc_udtf_default_sizer_1" query = f"select * from table({fn}(cursor(select i8 from {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(3,)] @@ -65,8 +65,8 @@ def test_python_fn_with_default_sizer(omnisci): @pytest.mark.parametrize("suffix,kind,expected", typs) -def test_default_sizer(omnisci, suffix, kind, expected): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5403") +def test_default_sizer(heavydb, suffix, kind, expected): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5403") codes = { "1": "i4", @@ -78,7 +78,7 @@ def test_default_sizer(omnisci, suffix, kind, expected): "R": "4", # another sizer } - table = omnisci.table_name + table = heavydb.table_name fn = f"ct_udtf_default_sizer{suffix}" query = f"select * from table({fn}(" for i, n in enumerate(kind): @@ -90,7 +90,7 @@ def test_default_sizer(omnisci, suffix, kind, expected): query += ", " if i + 1 < len(kind) else "" query += "));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) result = list(result) assert result == [(expected,)], (result, query, kind) diff --git a/rbc/tests/test_omnisci_column_list.py b/rbc/tests/heavyai/test_column_list.py similarity index 74% rename from rbc/tests/test_omnisci_column_list.py rename to rbc/tests/heavyai/test_column_list.py index a5c407e2..dd863c6e 100644 --- a/rbc/tests/test_omnisci_column_list.py +++ b/rbc/tests/heavyai/test_column_list.py @@ -1,5 +1,5 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture scalar_types = ['float64', 'int64', 'float32', 'int32', 'int16', 'int8'] table_columns_map = dict(int64='i8', int32='i4', int16='i2', int8='i1', @@ -9,16 +9,16 @@ @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('int32 columns_sum1(Cursor, ColumnList>,' + @heavydb('int32 columns_sum1(Cursor, ColumnList>,' ' RowMultiplier, OutputColumn)', T=scalar_types, devices=['cpu']) def columns_sum1(rowid, lst, m, out): @@ -28,7 +28,7 @@ def columns_sum1(rowid, lst, m, out): out[i] += lst.ptrs[j][i] return lst.size - @omnisci('int32 columns_sum2(Cursor, ColumnList>,' + @heavydb('int32 columns_sum2(Cursor, ColumnList>,' ' RowMultiplier, OutputColumn)', T=scalar_types, devices=['cpu']) def columns_sum2(rowid, lst, m, out): @@ -42,7 +42,7 @@ def columns_sum2(rowid, lst, m, out): out[i] += col[i] return lst.size - @omnisci('int32 columns_sum3(Cursor, ColumnList>,' + @heavydb('int32 columns_sum3(Cursor, ColumnList>,' ' RowMultiplier, OutputColumn)', T=scalar_types, devices=['cpu']) def columns_sum3(rowid, lst, m, out): @@ -55,19 +55,19 @@ def columns_sum3(rowid, lst, m, out): @pytest.mark.parametrize("variant", ['1', "2", "3"]) @pytest.mark.parametrize("T", scalar_types) -def test_columns_sum(omnisci, T, variant): +def test_columns_sum(heavydb, T, variant): v = table_columns_map[T] c = sql_type_map[T] - query = f'select rowid, ({v} + {v} + 1) from {omnisci.table_name} ORDER BY rowid' - _, result = omnisci.sql_execute(query) + query = f'select rowid, ({v} + {v} + 1) from {heavydb.table_name} ORDER BY rowid' + _, result = heavydb.sql_execute(query) expected = list(zip(*result))[1] - # using `.. ORDER BY rowid LIMIT ALL` will crash omniscidb server + # using `.. ORDER BY rowid LIMIT ALL` will crash heavydb server query = (f'select * from table(columns_sum{variant}(cursor(' f'select rowid, {v}, CAST({v}+1 as {c}) ' - f'from {omnisci.table_name} ORDER BY rowid LIMIT 1000), 1))') - _, result = omnisci.sql_execute(query) + f'from {heavydb.table_name} ORDER BY rowid LIMIT 1000), 1))') + _, result = heavydb.sql_execute(query) actual = list(zip(*result))[0] assert expected == actual diff --git a/rbc/tests/test_omnisci_column_null.py b/rbc/tests/heavyai/test_column_null.py similarity index 65% rename from rbc/tests/test_omnisci_column_null.py rename to rbc/tests/heavyai/test_column_null.py index 73e450dc..86b79eac 100644 --- a/rbc/tests/test_omnisci_column_null.py +++ b/rbc/tests/heavyai/test_column_null.py @@ -1,18 +1,18 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('int32(Column, RowMultiplier, OutputColumn)', + @heavydb('int32(Column, RowMultiplier, OutputColumn)', T=['int8', 'int16', 'int32', 'int64', 'float32', 'float64']) def my_row_copier_mul(x, m, y): input_row_count = len(x) @@ -24,7 +24,7 @@ def my_row_copier_mul(x, m, y): return m * input_row_count # cannot overload boolean8 and int8 as boolean8 is interpreted as int8 - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def my_row_copier_mul_bool(x, m, y): input_row_count = len(x) for i in range(input_row_count): @@ -41,38 +41,38 @@ def my_row_copier_mul_bool(x, m, y): @pytest.mark.parametrize('col', colnames) -def test_null_value(omnisci, col): +def test_null_value(heavydb, col): typ = dict(f4='float32', f8='float64', i1='int8', i2='int16', i4='int32', i8='int64', b='bool')[col] if typ == 'bool': prefix = '_bool' - descr, expected = omnisci.sql_execute( - f'select {col}, not {col} from {omnisci.table_name}null') + descr, expected = heavydb.sql_execute( + f'select {col}, not {col} from {heavydb.table_name}null') data, expected = zip(*list(expected)) else: prefix = '' - descr, expected = omnisci.sql_execute( - f'select {col}, {col} + {col} from {omnisci.table_name}null') + descr, expected = heavydb.sql_execute( + f'select {col}, {col} + {col} from {heavydb.table_name}null') data, expected = zip(*list(expected)) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'select * from table(my_row_copier_mul{prefix}(cursor(select {col} ' - f'from {omnisci.table_name}null), 1));') + f'from {heavydb.table_name}null), 1));') result, = zip(*list(result)) assert result == expected, (result, expected, data) -def test_row_adder(omnisci): - descr, expected = omnisci.sql_execute( - f'select f8, f8 + f8 from {omnisci.table_name}null') +def test_row_adder(heavydb): + descr, expected = heavydb.sql_execute( + f'select f8, f8 + f8 from {heavydb.table_name}null') data, expected = zip(*list(expected)) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select * from table(row_adder(1, cursor(select f8, f8 ' - f'from {omnisci.table_name}null)))') + f'from {heavydb.table_name}null)))') result, = zip(*list(result)) assert result == expected, (result, expected, data) diff --git a/rbc/tests/test_omnisci_column_runtime_alloc.py b/rbc/tests/heavyai/test_column_runtime_alloc.py similarity index 68% rename from rbc/tests/test_omnisci_column_runtime_alloc.py rename to rbc/tests/heavyai/test_column_runtime_alloc.py index 6602335d..de55d4e0 100644 --- a/rbc/tests/test_omnisci_column_runtime_alloc.py +++ b/rbc/tests/heavyai/test_column_runtime_alloc.py @@ -1,6 +1,6 @@ import pytest -from rbc.tests import omnisci_fixture -from rbc.externals.omniscidb import set_output_row_size +from rbc.tests import heavydb_fixture +from rbc.externals.heavydb import set_output_row_size scalar_types = ['float64', 'int64', 'float32', 'int32', 'int16', 'int8'] @@ -9,16 +9,16 @@ @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 7, 1)): + for o in heavydb_fixture(globals(), minimal_version=(5, 7, 1)): define(o) yield o -def define(omnisci): +def define(heavydb): - @omnisci('int32(Column, OutputColumn)', T=scalar_types, devices=['cpu']) + @heavydb('int32(Column, OutputColumn)', T=scalar_types, devices=['cpu']) def column_set_output_row_size(inp, out): sz = len(inp) set_output_row_size(sz) @@ -26,7 +26,7 @@ def column_set_output_row_size(inp, out): out[i] = inp[i] return sz - @omnisci('int32(ColumnList, OutputColumn)', T=['int32'], devices=['cpu']) + @heavydb('int32(ColumnList, OutputColumn)', T=['int32'], devices=['cpu']) def columnlist_set_output_row_size(lst, out): set_output_row_size(lst.ncols) for j in range(lst.ncols): @@ -38,14 +38,14 @@ def columnlist_set_output_row_size(lst, out): @pytest.mark.parametrize('T', table_columns_map) -def test_column_alloc(omnisci, T): +def test_column_alloc(heavydb, T): fn = 'column_set_output_row_size' col = table_columns_map[T] query = (f'select * from table({fn}(cursor(' - f'select {col} from {omnisci.table_name})));') + f'select {col} from {heavydb.table_name})));') - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) if 'f' in col: expected = [(0.0,), (1.0,), (2.0,), (3.0,), (4.0,)] @@ -55,11 +55,11 @@ def test_column_alloc(omnisci, T): @pytest.mark.parametrize("fn", ('column_list_row_sum', 'columnlist_set_output_row_size')) -def test_columnlist_alloc(omnisci, fn): +def test_columnlist_alloc(heavydb, fn): query = (f'select * from table({fn}(cursor(' - f'select i4, i4+1, i4+2 from {omnisci.table_name})));') + f'select i4, i4+1, i4+2 from {heavydb.table_name})));') - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(10,), (15,), (20,)] diff --git a/rbc/tests/test_omnisci_constants.py b/rbc/tests/heavyai/test_constants.py similarity index 72% rename from rbc/tests/test_omnisci_constants.py rename to rbc/tests/heavyai/test_constants.py index bf5bf0e0..97c66d48 100644 --- a/rbc/tests/test_omnisci_constants.py +++ b/rbc/tests/heavyai/test_constants.py @@ -1,20 +1,20 @@ import pytest import numpy as np -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.stdlib import array_api from collections import OrderedDict @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), load_test_data=False): + for o in heavydb_fixture(globals(), load_test_data=False): define(o) yield o -def define(omnisci): - @omnisci('float64(int32)') +def define(heavydb): + @heavydb('float64(int32)') def get_constant(typ): if typ == 0: return array_api.e @@ -29,9 +29,9 @@ def get_constant(typ): @pytest.mark.parametrize('C', constants_map) -def test_constants(omnisci, C): +def test_constants(heavydb, C): idx = list(constants_map.keys()).index(C) - _, result = omnisci.sql_execute(f'select get_constant({idx});') + _, result = heavydb.sql_execute(f'select get_constant({idx});') expected = constants_map[C] if np.isnan(expected): diff --git a/rbc/tests/test_omnisci_device_selection.py b/rbc/tests/heavyai/test_device_selection.py similarity index 73% rename from rbc/tests/test_omnisci_device_selection.py rename to rbc/tests/heavyai/test_device_selection.py index 1c64d156..02dc33cd 100644 --- a/rbc/tests/test_omnisci_device_selection.py +++ b/rbc/tests/heavyai/test_device_selection.py @@ -1,11 +1,11 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): define(o) def require_loadtime(kind, _cache=[None]): @@ -44,24 +44,24 @@ def decode(c): err_device_code: 'err'}[c] -def get_single_result(omnisci, kind, func): +def get_single_result(heavydb, kind, func): if func == 'any': if kind == 'rt': return any_device_code - return gpu_device_code if omnisci.has_cuda else cpu_device_code + return gpu_device_code if heavydb.has_cuda else cpu_device_code if func == 'cpu': return cpu_device_code if func == 'gpu': - return gpu_device_code if omnisci.has_cuda else err_device_code + return gpu_device_code if heavydb.has_cuda else err_device_code if func == 'both': - return gpu_device_code if omnisci.has_cuda else cpu_device_code + return gpu_device_code if heavydb.has_cuda else cpu_device_code raise NotImplementedError(repr((kind, func))) -def get_pair_result(omnisci, ext, kind1, func1, kind2, func2, +def get_pair_result(heavydb, ext, kind1, func1, kind2, func2, allow_query_step_cpu_retry=False): - r1 = get_single_result(omnisci, kind1, func1) - r2 = get_single_result(omnisci, kind2, func2) + r1 = get_single_result(heavydb, kind1, func1) + r2 = get_single_result(heavydb, kind2, func2) if r1 == err_device_code or r2 == err_device_code: return err_device_code if allow_query_step_cpu_retry and ext in {'udtf/udf', 'udf/udtf'}: @@ -78,9 +78,9 @@ def get_pair_result(omnisci, ext, kind1, func1, kind2, func2, return r1, r2 -def execute1(omnisci, query): +def execute1(heavydb, query): try: - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) except Exception as msg: return msg else: @@ -88,9 +88,9 @@ def execute1(omnisci, query): return decode(result[0][0]) -def execute2(omnisci, query): +def execute2(heavydb, query): try: - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) except Exception as msg: return msg else: @@ -98,21 +98,21 @@ def execute2(omnisci, query): return decode(result[0][0]), decode(result[0][1]) -def get_worker1(omnisci, ext, kind, func): - expected = decode(get_single_result(omnisci, kind, func)) +def get_worker1(heavydb, ext, kind, func): + expected = decode(get_single_result(heavydb, kind, func)) if ext == 'udf': query = f'select {kind}_device_selection_{ext}_{func}(0)' return execute1, query, expected if ext == 'udtf': query = (f'select out0 from table({kind}_device_selection_{ext}_{func}' - f'(cursor(select i4 from {omnisci.table_name})))') + f'(cursor(select i4 from {heavydb.table_name})))') return execute1, query, expected raise NotImplementedError(repr((ext, kind, func))) -def get_worker2(omnisci, ext, kind1, func1, kind2, func2, +def get_worker2(heavydb, ext, kind1, func1, kind2, func2, allow_query_step_cpu_retry=False): - expected = get_pair_result(omnisci, ext, kind1, func1, kind2, func2, + expected = get_pair_result(heavydb, ext, kind1, func1, kind2, func2, allow_query_step_cpu_retry=allow_query_step_cpu_retry) if isinstance(expected, tuple): expected = tuple(map(decode, expected)) @@ -125,56 +125,56 @@ def get_worker2(omnisci, ext, kind1, func1, kind2, func2, if ext == 'udtf/udf': query = (f'select out0 from table({kind1}_device_selection_udtf_{func1}' f'(cursor(select {kind2}_device_selection_udf_{func2}(i4)' - f' from {omnisci.table_name})))') + f' from {heavydb.table_name})))') return execute1, query, expected[0] if isinstance(expected, tuple) else expected raise NotImplementedError(repr((ext, kind1, func1, kind2, func2))) -def define(omnisci): +def define(heavydb): - @omnisci('int32(int32)') + @heavydb('int32(int32)') def rt_device_selection_udf_any(x): # cannot determine which device is actually used return any_device_code - @omnisci('int32(int32)', devices=['cpu']) + @heavydb('int32(int32)', devices=['cpu']) def rt_device_selection_udf_cpu(x): return cpu_device_code - @omnisci('int32(int32)', devices=['gpu']) + @heavydb('int32(int32)', devices=['gpu']) def rt_device_selection_udf_gpu(x): return gpu_device_code - @omnisci('int32(int32)', devices=['cpu']) # NOQA + @heavydb('int32(int32)', devices=['cpu']) # NOQA def rt_device_selection_udf_both(x): # NOQA return cpu_device_code - @omnisci('int32(int32)', devices=['gpu']) # NOQA + @heavydb('int32(int32)', devices=['gpu']) # NOQA def rt_device_selection_udf_both(x): # NOQA return gpu_device_code - @omnisci('int32(Column, OutputColumn)') + @heavydb('int32(Column, OutputColumn)') def rt_device_selection_udtf_any(x, out): # cannot determine which device is actually used out[0] = any_device_code return 1 - @omnisci('int32(Column, OutputColumn)', devices=['cpu']) + @heavydb('int32(Column, OutputColumn)', devices=['cpu']) def rt_device_selection_udtf_cpu(x, out): out[0] = cpu_device_code return 1 - @omnisci('int32(Column, OutputColumn)', devices=['gpu']) + @heavydb('int32(Column, OutputColumn)', devices=['gpu']) def rt_device_selection_udtf_gpu(x, out): out[0] = gpu_device_code return 1 - @omnisci('int32(Column, OutputColumn)', devices=['cpu']) # NOQA + @heavydb('int32(Column, OutputColumn)', devices=['cpu']) # NOQA def rt_device_selection_udtf_both(x, out): # NOQA out[0] = cpu_device_code return 1 - @omnisci('int32(Column, OutputColumn)', devices=['gpu']) # NOQA + @heavydb('int32(Column, OutputColumn)', devices=['gpu']) # NOQA def rt_device_selection_udtf_both(x, out): # NOQA out[0] = gpu_device_code return 1 @@ -183,14 +183,14 @@ def rt_device_selection_udtf_both(x, out): # NOQA @pytest.mark.parametrize("func", funcs) @pytest.mark.parametrize("ext", ['udf', 'udtf']) @pytest.mark.parametrize("kind", ['rt', 'ct', 'lt']) -def test_device_selection_single(omnisci, func, ext, kind): - omnisci.require_loadtime(kind) +def test_device_selection_single(heavydb, func, ext, kind): + heavydb.require_loadtime(kind) if kind == 'lt' and ext == 'udtf': pytest.skip('Load-time UDTFs not supported') - execute, query, expected = get_worker1(omnisci, ext, kind, func) - result = execute(omnisci, query) + execute, query, expected = get_worker1(heavydb, ext, kind, func) + result = execute(heavydb, query) if isinstance(result, Exception): assert expected == decode(err_device_code), str(result) else: @@ -203,24 +203,24 @@ def test_device_selection_single(omnisci, func, ext, kind): @pytest.mark.parametrize("ext", ['udf/udf', 'udtf/udf']) @pytest.mark.parametrize("kind2", kinds) @pytest.mark.parametrize("kind1", kinds) -def test_device_selection_pair(omnisci, func12, ext, kind2, kind1): +def test_device_selection_pair(heavydb, func12, ext, kind2, kind1): func12 = tuple(func12.split('/')) func1, func2 = func12 - omnisci.require_loadtime(kind1) - omnisci.require_loadtime(kind2) + heavydb.require_loadtime(kind1) + heavydb.require_loadtime(kind2) if 'udtf' in ext and kind1 == 'lt': pytest.skip('Load-time UDTFs not supported') # Omniscidb server option --allow-query-step-cpu-retry=1 was - # introduced in omniscidb-internal PR 5798, here assuming the + # introduced in heavydb-internal PR 5798, here assuming the # default value. - allow_query_step_cpu_retry = omnisci.version[:2] >= (5, 8) + allow_query_step_cpu_retry = heavydb.version[:2] >= (5, 8) execute, query, expected = get_worker2( - omnisci, ext, kind1, func1, kind2, func2, + heavydb, ext, kind1, func1, kind2, func2, allow_query_step_cpu_retry=allow_query_step_cpu_retry) - result = execute(omnisci, query) + result = execute(heavydb, query) if isinstance(result, Exception): assert expected == decode(err_device_code), str(result) else: diff --git a/rbc/tests/test_omnisci_external.py b/rbc/tests/heavyai/test_external.py similarity index 64% rename from rbc/tests/test_omnisci_external.py rename to rbc/tests/heavyai/test_external.py index 1336e047..a1ab6c34 100644 --- a/rbc/tests/test_omnisci_external.py +++ b/rbc/tests/heavyai/test_external.py @@ -1,28 +1,28 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.external import external from numba import types @pytest.fixture(scope="module") -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): yield o -def test_valid_signatures(omnisci): +def test_valid_signatures(heavydb): external("f64 log2(f64)") assert True -def test_invalid_signature(omnisci): +def test_invalid_signature(heavydb): with pytest.raises(ValueError) as excinfo: external(types.int64) assert "signature must represent a function type" in str(excinfo) -def test_unnamed_external(omnisci): +def test_unnamed_external(heavydb): with pytest.raises(ValueError) as excinfo: external("f64(f64)") @@ -34,28 +34,28 @@ def test_unnamed_external(omnisci): assert "external function name not specified for signature" in str(excinfo) -def test_replace_declaration(omnisci): +def test_replace_declaration(heavydb): _ = external("f64 fma(f64)|CPU") fma = external("f64 fma(f64, f64, f64)|CPU") - @omnisci("double(double, double, double)", devices=["cpu"]) + @heavydb("double(double, double, double)", devices=["cpu"]) def test_fma(a, b, c): return fma(a, b, c) - _, result = omnisci.sql_execute("select test_fma(1.0, 2.0, 3.0)") + _, result = heavydb.sql_execute("select test_fma(1.0, 2.0, 3.0)") assert list(result) == [(5.0,)] -def test_require_target_info(omnisci): +def test_require_target_info(heavydb): log2 = external("double log2(double)|CPU") - @omnisci("double(double)", devices=["cpu"]) + @heavydb("double(double)", devices=["cpu"]) def test_log2(a): return log2(a) - _, result = omnisci.sql_execute("select test_log2(8.0)") + _, result = heavydb.sql_execute("select test_log2(8.0)") assert list(result) == [(3.0,)] diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/heavyai/test_heavyai.py similarity index 74% rename from rbc/tests/test_omnisci.py rename to rbc/tests/heavyai/test_heavyai.py index 916dd848..fc531f0b 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/heavyai/test_heavyai.py @@ -4,11 +4,11 @@ import numpy as np from rbc.errors import UnsupportedError, OmnisciServerError -from rbc.tests import omnisci_fixture, assert_equal +from rbc.tests import heavydb_fixture, assert_equal from rbc.typesystem import Type -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() # Throw an error on Travis CI if the server is not available if "TRAVIS" in os.environ and not available_version: pytest.exit(msg=reason, returncode=1) @@ -43,14 +43,14 @@ def test_get_client_config(tmpdir, heavydb_brand): os.environ[client_conf_env] = conf_file try: - conf = rbc_omnisci.get_client_config() + conf = rbc_heavydb.get_client_config() assert conf['user'] == 'foo' assert conf['password'] == 'secret' assert conf['port'] == 1234 assert conf['host'] == 'example.com' assert conf['dbname'] == heavydb_brand assert conf['debug'] == bool(0) - conf = rbc_omnisci.get_client_config(dbname='test') + conf = rbc_heavydb.get_client_config(dbname='test') assert conf['dbname'] == 'test' finally: if old_conf is None: @@ -66,61 +66,61 @@ def nb_version(): @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): yield o -def test_direct_call(omnisci): - omnisci.reset() +def test_direct_call(heavydb): + heavydb.reset() - @omnisci('double(double)') + @heavydb('double(double)') def farhenheit2celcius(f): return (f - 32) * 5 / 9 assert_equal(farhenheit2celcius(40).execute(), np.float32(40 / 9)) -def test_local_caller(omnisci): - omnisci.reset() +def test_local_caller(heavydb): + heavydb.reset() def func(f): return f - caller = omnisci('double(double)')(func) + caller = heavydb('double(double)')(func) msg = "Cannot create a local `Caller`" with pytest.raises(UnsupportedError, match=msg): _ = caller.local -def test_redefine(omnisci): - omnisci.reset() +def test_redefine(heavydb): + heavydb.reset() - @omnisci('i32(i32)') + @heavydb('i32(i32)') def incr(x): return x + 1 - desrc, result = omnisci.sql_execute( - f'select i4, incr(i4) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select i4, incr(i4) from {heavydb.table_name}') for x, x1 in result: assert x1 == x + 1 # Re-defining triggers a warning message when in debug mode - @omnisci('i32(i32)') # noqa: F811 + @heavydb('i32(i32)') # noqa: F811 def incr(x): # noqa: F811 return x + 2 - desrc, result = omnisci.sql_execute( - f'select i4, incr(i4) from {omnisci.table_name}') + desrc, result = heavydb.sql_execute( + f'select i4, incr(i4) from {heavydb.table_name}') for x, x1 in result: assert x1 == x + 2 -def test_single_argument_overloading(omnisci): - omnisci.reset() +def test_single_argument_overloading(heavydb): + heavydb.reset() - @omnisci( + @heavydb( 'f64(f64)', 'i64(i64)', 'i32(i32)', @@ -129,22 +129,22 @@ def test_single_argument_overloading(omnisci): ) def mydecr(x): return x - 1 - desrc, result = omnisci.sql_execute( - 'select f4, mydecr(f4) from {omnisci.table_name}'.format(**locals())) + desrc, result = heavydb.sql_execute( + 'select f4, mydecr(f4) from {heavydb.table_name}'.format(**locals())) result = list(result) assert len(result) > 0 for x, x1 in result: assert x1 == x - 1 assert isinstance(x1, type(x)) - desrc, result = omnisci.sql_execute( - 'select f8, mydecr(f8) from {omnisci.table_name}'.format(**locals())) + desrc, result = heavydb.sql_execute( + 'select f8, mydecr(f8) from {heavydb.table_name}'.format(**locals())) result = list(result) assert len(result) > 0 for x, x1 in result: assert x1 == x - 1 assert isinstance(x1, type(x)) - desrc, result = omnisci.sql_execute( - 'select i4, mydecr(i4) from {omnisci.table_name}'.format(**locals())) + desrc, result = heavydb.sql_execute( + 'select i4, mydecr(i4) from {heavydb.table_name}'.format(**locals())) result = list(result) assert len(result) > 0 for x, x1 in result: @@ -152,17 +152,17 @@ def mydecr(x): assert isinstance(x1, type(x)) -def test_thrift_api_doc(omnisci): - omnisci.reset() +def test_thrift_api_doc(heavydb): + heavydb.reset() - @omnisci('double(int, double)', + @heavydb('double(int, double)', 'float(int, float)', 'int(int, int)') def foo(i, v): return v * i + 55 - descr, result = omnisci.sql_execute( - 'select f8, foo(i4, f8) from {omnisci.table_name}'.format(**locals())) + descr, result = heavydb.sql_execute( + 'select f8, foo(i4, f8) from {heavydb.table_name}'.format(**locals())) result = list(result) assert len(result) > 0 for i, (x, x1) in enumerate(result): @@ -170,39 +170,39 @@ def foo(i, v): assert isinstance(x1, type(x)) -def test_multiple_implementation(omnisci): - omnisci.reset() +def test_multiple_implementation(heavydb): + heavydb.reset() - @omnisci('int(f64)', 'int(i64)') # noqa: F811 + @heavydb('int(f64)', 'int(i64)') # noqa: F811 def bits(x): return 64 - @omnisci('int(f32)', 'int(i32)') # noqa: F811 + @heavydb('int(f32)', 'int(i32)') # noqa: F811 def bits(x): # noqa: F811 return 32 - @omnisci('int(i16)') # noqa: F811 + @heavydb('int(i16)') # noqa: F811 def bits(x): # noqa: F811 return 16 - @omnisci('int(i8)') # noqa: F811 + @heavydb('int(i8)') # noqa: F811 def bits(x): # noqa: F811 return 8 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(i1), bits(i2), bits(i4), bits(f4), bits(i8), bits(f8)' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) result = list(result) assert len(result) == 1 assert result[0] == (8, 16, 32, 32, 64, 64) -def test_loadtime_udf(omnisci): - # This test requires that omnisci server is started with `--udf +def test_loadtime_udf(heavydb): + # This test requires that heavydb server is started with `--udf # sample_udf.cpp` option where the .cpp file defines `udf_diff` try: - desrc, result = omnisci.sql_execute( - 'select i4, udf_diff2(i4, i4) from {omnisci.table_name}' + desrc, result = heavydb.sql_execute( + 'select i4, udf_diff2(i4, i4) from {heavydb.table_name}' .format(**locals())) except Exception as msg: assert "Undefined function call 'udf_diff2'" in str(msg) @@ -213,44 +213,44 @@ def test_loadtime_udf(omnisci): assert d == 0 -def test_f32(omnisci): +def test_f32(heavydb): """If UDF name ends with an underscore, expect strange behaviour. For instance, defining - @omnisci('f32(f32)', 'f32(f64)') + @heavydb('f32(f32)', 'f32(f64)') def f32_(x): return x+4.5 the query `select f32_(0.0E0))` fails but not when defining - @omnisci('f32(f64)', 'f32(f32)') + @heavydb('f32(f64)', 'f32(f32)') def f32_(x): return x+4.5 - (notice the order of signatures in omnisci decorator argument). + (notice the order of signatures in heavydb decorator argument). """ - @omnisci('f32(f32)', 'f32(f64)') # noqa: F811 + @heavydb('f32(f32)', 'f32(f64)') # noqa: F811 def f_32(x): return x+4.5 - descr, result = omnisci.sql_execute( - 'select f_32(0.0E0) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select f_32(0.0E0) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (4.5,) -def test_castop(omnisci): - @omnisci('i16(i16)') # noqa: F811 +def test_castop(heavydb): + @heavydb('i16(i16)') # noqa: F811 def i32(x): return x+2 # noqa: F811 - @omnisci('i32(i32)') # noqa: F811 + @heavydb('i32(i32)') # noqa: F811 def i32(x): return x+4 # noqa: F811 - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i32(cast(1 as int))' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (5,) -def test_binding(omnisci): - omnisci.reset() +def test_binding(heavydb): + heavydb.reset() full = os.environ.get('RBC_TESTS_FULL', 'FALSE').lower() in ['1', 'true', 'on'] if full: @@ -272,7 +272,7 @@ def test_binding(omnisci): column_vars_types = argument_types if available_version[:2] >= (5, 9): - omnisci.require_version((5, 9), 'Requires omniscidb-internal PR 6003') + heavydb.require_version((5, 9), 'Requires heavydb-internal PR 6003') def get_result(overload_types, input_type, is_literal): overload_types_ = overload_types[::-1 if is_literal else 1] @@ -317,7 +317,7 @@ def get_result(overload_types, input_type, is_literal): assert len(literals) == len(literals_types) assert len(column_vars) == len(column_vars_types) - omnisci.reset() + heavydb.reset() functions = [] for r in range(1, len(argument_types) + 1): if not full and r not in [1, 2, len(argument_types)]: @@ -327,14 +327,14 @@ def get_result(overload_types, input_type, is_literal): for i in p: foo = eval(f'lambda x: {i}') foo.__name__ = fname - omnisci(f'i8({argument_types[i]})')(foo) + heavydb(f'i8({argument_types[i]})')(foo) functions.append(fname) - omnisci.register() + heavydb.register() for fname in functions: for input, input_type in zip(literals, literals_types): try: - descr, result = omnisci.sql_execute(f'select {fname}({input})') + descr, result = heavydb.sql_execute(f'select {fname}({input})') result = list(result) result = argument_types[result[0][0]] except Exception: @@ -344,8 +344,8 @@ def get_result(overload_types, input_type, is_literal): for input, input_type in zip(column_vars, column_vars_types): try: - descr, result = omnisci.sql_execute( - f'select {fname}({input}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {fname}({input}) from {heavydb.table_name} limit 1') result = list(result) result = argument_types[result[0][0]] except Exception: @@ -354,7 +354,7 @@ def get_result(overload_types, input_type, is_literal): assert result == expected, (fname, input, input_type, result, expected) -def test_casting(omnisci): +def test_casting(heavydb): """Define UDFs: i8(), i16(), i32(), i64() f32(), f64() @@ -401,55 +401,55 @@ def test_casting(omnisci): test_binding is superior test with respect to successful UDF executions but it does not check exception messages. """ - omnisci.reset() + heavydb.reset() - @omnisci('i8(i8)') # noqa: F811 + @heavydb('i8(i8)') # noqa: F811 def i8(x): return x+1 - @omnisci('i16(i16)') # noqa: F811 + @heavydb('i16(i16)') # noqa: F811 def i16(x): return x+2 - @omnisci('i32(i32)') # noqa: F811 + @heavydb('i32(i32)') # noqa: F811 def i32(x): return x+4 - @omnisci('i64(i64)') # noqa: F811 + @heavydb('i64(i64)') # noqa: F811 def i64(x): return x+8 - @omnisci('f32(f32)') # noqa: F811 + @heavydb('f32(f32)') # noqa: F811 def f32(x): return x+4.5 # cannot create a 8-bit and 16-bit int literals in sql, so, using # the following helper functions: - @omnisci('i8(i8)', 'i8(i16)', 'i8(i32)') # noqa: F811 + @heavydb('i8(i8)', 'i8(i16)', 'i8(i32)') # noqa: F811 def i_8(x): return x+1 - @omnisci('i16(i16)', 'i16(i32)') # noqa: F811 + @heavydb('i16(i16)', 'i16(i32)') # noqa: F811 def i_16(x): return x+2 # cannot create a 32-bit float literal in sql, so, using a helper # function for that: - @omnisci('f32(f32)', 'f32(f64)') # noqa: F811 + @heavydb('f32(f32)', 'f32(f64)') # noqa: F811 def f_32(x): return x+4.5 - @omnisci('f64(f64)') # noqa: F811 + @heavydb('f64(f64)') # noqa: F811 def f64(x): return x+8.5 - @omnisci('i8(i8)') + @heavydb('i8(i8)') def ifoo(x): return x + 1 - @omnisci('i16(i16)') # noqa: F811 + @heavydb('i16(i16)') # noqa: F811 def ifoo(x): return x + 2 # noqa: F811 - @omnisci('i32(i32)') # noqa: F811 + @heavydb('i32(i32)') # noqa: F811 def ifoo(x): return x + 4 # noqa: F811 - @omnisci('i64(i64)') # noqa: F811 + @heavydb('i64(i64)') # noqa: F811 def ifoo(x): return x + 8 # noqa: F811 - @omnisci('f32(f32)') + @heavydb('f32(f32)') def ffoo(x): return x + 4.5 # noqa: F811 - @omnisci('f64(f64)') # noqa: F811 + @heavydb('f64(f64)') # noqa: F811 def ffoo(x): return x + 8.5 # noqa: F811 rows = [] @@ -469,8 +469,8 @@ def ffoo(x): return x + 8.5 # noqa: F811 ('double', 'f64')]: cols.append(f'{func:4}') try: - descr, result = omnisci.sql_execute( - f'select {func}({ivalue}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {func}({ivalue}) from {heavydb.table_name} limit 1') status = 'OK' except Exception: status = 'FAIL' @@ -483,106 +483,106 @@ def ffoo(x): return x + 8.5 # noqa: F811 print('\n\nSUPPORTED CASTING RULES FOR SCALAR ARGUMENTS:\n') print('\n'.join(rows)) - descr, result = omnisci.sql_execute( - 'select i_8(0),i_16(0),i32(0),i64(0) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select i_8(0),i_16(0),i32(0),i64(0) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (1, 2, 4, 8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i_8(i1),i16(i2),i32(i4),i64(i8)' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (1, 2, 4, 8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select ifoo(i_8(0)),ifoo(i_16(0)),ifoo(i32(0)),ifoo(i64(0))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (1+1, 2+2, 4+4, 8+8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select ifoo(i1),ifoo(i2),ifoo(i4),ifoo(i8)' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (1, 2, 4, 8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i64(i_8(0)), i64(i_16(0)),i64(i32(0)),i64(i64(0))' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (9, 10, 12, 16) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i64(i1), i64(i2),i64(i4),i64(i8)' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (8, 8, 8, 8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i32(i_8(0)), i32(i_16(0)),i32(i32(0))' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (5, 6, 8) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select i32(i1), i32(i2),i32(i4)' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (4, 4, 4) - descr, result = omnisci.sql_execute( - 'select i16(i_8(0)), i16(i_16(0)) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select i16(i_8(0)), i16(i_16(0)) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (3, 4) - descr, result = omnisci.sql_execute( - 'select i16(i1), i16(i2) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select i16(i1), i16(i2) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (2, 2) - descr, result = omnisci.sql_execute( - 'select i8(i_8(0)) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select i8(i_8(0)) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (2,) - descr, result = omnisci.sql_execute( - 'select i8(i1) from {omnisci.table_name} limit 1'.format(**locals())) + descr, result = heavydb.sql_execute( + 'select i8(i1) from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (1,) - descr, result = omnisci.sql_execute( - 'select f_32(0.0E0), f64(0.0E0) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select f_32(0.0E0), f64(0.0E0) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (4.5, 8.5) - descr, result = omnisci.sql_execute( - 'select f_32(f4), f64(f8) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select f_32(f4), f64(f8) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (4.5, 8.5) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select f64(f_32(0.0E0)), f64(f64(0.0E0))' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (13.0, 17.0) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select f64(f4), f64(f8)' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (8.5, 8.5) - descr, result = omnisci.sql_execute( - 'select f32(f_32(0.0E0)) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select f32(f_32(0.0E0)) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (9.0,) - descr, result = omnisci.sql_execute( - 'select f32(f4) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select f32(f4) from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (4.5,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select ffoo(f_32(0.0E0)), ffoo(f64(0.0E0))' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (9.0, 17.0) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select ffoo(f4), ffoo(f8)' - ' from {omnisci.table_name} limit 1'.format(**locals())) + ' from {heavydb.table_name} limit 1'.format(**locals())) assert list(result)[0] == (4.5, 8.5) for v, f, t in [ @@ -601,11 +601,11 @@ def ffoo(x): return x + 8.5 # noqa: F811 ('f64(0)', r'f32', r'DOUBLE'), ('f8', r'f32', r'DOUBLE'), ]: - q = f'select {f}({v}) from {omnisci.table_name} limit 1' + q = f'select {f}({v}) from {heavydb.table_name} limit 1' match = (r".*(Function "+f+r"\("+t+r"\) not supported" r"|Could not bind "+f+r"\("+t+r"\))") with pytest.raises(Exception, match=match): - descr, result = omnisci.sql_execute(q) + descr, result = heavydb.sql_execute(q) print('query: ', q) print('expected: ', match) @@ -621,32 +621,32 @@ def ffoo(x): return x + 8.5 # noqa: F811 (r'TINYINT', 'i1', (8.5,)), ]: if available_version[:2] >= (5, 9): - # omniscidb-internal PR 6003 changed the casting table + # heavydb-internal PR 6003 changed the casting table if f == r'f32': r = r[0] - 4, - descr, result = omnisci.sql_execute( - f'select {f}({av}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {f}({av}) from {heavydb.table_name} limit 1') assert list(result)[0] == r, (f, at, av, r) elif available_version[:2] == (5, 8): - # omniscidb-internal PR 5814 changed the casting table + # heavydb-internal PR 5814 changed the casting table if f == r'f32' and at == r'BIGINT': with pytest.raises( Exception, match=(r".*(Function "+f+r"\("+at+r"\) not supported" r"|Could not bind "+f+r"\("+at+r"\))")): - descr, result = omnisci.sql_execute( - f'select {f}({av}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {f}({av}) from {heavydb.table_name} limit 1') else: if f == r'f32': r = r[0] - 4, - descr, result = omnisci.sql_execute( - f'select {f}({av}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {f}({av}) from {heavydb.table_name} limit 1') assert list(result)[0] == r, (f, at, av, r) elif f == r'f64': # temporary: allow integers as double arguments - descr, result = omnisci.sql_execute( - f'select {f}({av}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {f}({av}) from {heavydb.table_name} limit 1') assert list(result)[0] == r else: @@ -654,8 +654,8 @@ def ffoo(x): return x + 8.5 # noqa: F811 Exception, match=(r".*(Function "+f+r"\("+at+r"\) not supported" r"|Could not bind "+f+r"\("+at+r"\))")): - descr, result = omnisci.sql_execute( - 'select '+f+'('+av+f') from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + 'select '+f+'('+av+f') from {heavydb.table_name} limit 1') for f in [r'i64', r'i32', r'i16', r'i8']: for at, av in [ @@ -668,103 +668,103 @@ def ffoo(x): return x + 8.5 # noqa: F811 Exception, match=(r".*(Function "+f+r"\("+at+r"\) not supported" r"|Could not bind "+f+r"\("+at+r"\))")): - descr, result = omnisci.sql_execute( - f'select {f}({av}) from {omnisci.table_name} limit 1') + descr, result = heavydb.sql_execute( + f'select {f}({av}) from {heavydb.table_name} limit 1') -def test_truncate_issue(omnisci): - omnisci.reset() +def test_truncate_issue(heavydb): + heavydb.reset() - @omnisci('int(f64)', 'int(i64)') # noqa: F811 + @heavydb('int(f64)', 'int(i64)') # noqa: F811 def bits(x): # noqa: F811 return 64 - @omnisci('int(f32)', 'int(i32)') # noqa: F811 + @heavydb('int(f32)', 'int(i32)') # noqa: F811 def bits(x): # noqa: F811 return 32 - @omnisci('int(i16)') # noqa: F811 + @heavydb('int(i16)') # noqa: F811 def bits(x): # noqa: F811 return 16 - @omnisci('int(i8)') # noqa: F811 + @heavydb('int(i8)') # noqa: F811 def bits(x): # noqa: F811 return 8 - descr, result = omnisci.sql_execute( - 'select bits(truncate(2016, 1)) from {omnisci.table_name} limit 1' + descr, result = heavydb.sql_execute( + 'select bits(truncate(2016, 1)) from {heavydb.table_name} limit 1' .format(**locals())) if available_version[:2] >= (5, 8): - # omniscidb-internal PR 5915 changes the casting table + # heavydb-internal PR 5915 changes the casting table assert list(result)[0] in [(64,)] else: assert list(result)[0] in [(16,), (32,)] - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(truncate(cast(2016.0 as smallint), 1))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) if available_version[:2] >= (5, 8): assert list(result)[0] == (64,) else: assert list(result)[0] == (16,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(truncate(cast(2016.0 as int), 1))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) if available_version[:2] >= (5, 8): assert list(result)[0] == (64,) else: assert list(result)[0] == (32,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(truncate(cast(2016.0 as bigint), 1))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (64,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(truncate(cast(2016.0 as float), 1))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) if available_version[:2] >= (5, 8): assert list(result)[0] == (64,) else: assert list(result)[0] == (32,) - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( 'select bits(truncate(cast(2016.0 as double), 1))' - ' from {omnisci.table_name} limit 1' + ' from {heavydb.table_name} limit 1' .format(**locals())) assert list(result)[0] == (64,) -def test_unregistering(omnisci): - omnisci.reset() +def test_unregistering(heavydb): + heavydb.reset() - @omnisci('i32(i32)') + @heavydb('i32(i32)') def fahrenheit2celsius(f): return (f - 32) * 5 / 9 - _, result = omnisci.sql_execute('select fahrenheit2celsius(40)') + _, result = heavydb.sql_execute('select fahrenheit2celsius(40)') assert list(result)[0] == (4,) - omnisci.unregister() + heavydb.unregister() msg = "Undefined function call" with pytest.raises(OmnisciServerError, match=msg): - omnisci.sql_execute('select fahrenheit2celsius(40)') + heavydb.sql_execute('select fahrenheit2celsius(40)') -def test_format_type(omnisci): +def test_format_type(heavydb): def test(s, caller=False): - with omnisci.targets['cpu']: - with Type.alias(**omnisci.typesystem_aliases): + with heavydb.targets['cpu']: + with Type.alias(**heavydb.typesystem_aliases): typ = Type.fromobject(s) if caller: - typ = omnisci.caller_signature(typ) - return omnisci.format_type(typ) + typ = heavydb.caller_signature(typ) + return heavydb.format_type(typ) assert test('int32 x') == 'int32 x' assert test('Column') == 'Column' diff --git a/rbc/tests/test_omnisci_math.py b/rbc/tests/heavyai/test_math.py similarity index 92% rename from rbc/tests/test_omnisci_math.py rename to rbc/tests/heavyai/test_math.py index a31c37dc..ff05d309 100644 --- a/rbc/tests/test_omnisci_math.py +++ b/rbc/tests/heavyai/test_math.py @@ -3,10 +3,11 @@ import sys import numpy as np import numba as nb -import rbc.omniscidb as rbc_omnisci + +import rbc.heavydb as rbc_heavydb from rbc.stdlib import array_api -available_version, reason = rbc_omnisci.is_available() +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @@ -17,11 +18,11 @@ def nb_version(): @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) - table_name = 'rbc_test_omnisci_math' +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) + table_name = 'rbc_test_heavydb_math' m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') @@ -123,11 +124,11 @@ def omnisci(): @pytest.mark.parametrize("device", devices) @pytest.mark.parametrize("fn_name, signature", math_functions, ids=["math." + item[0] for item in math_functions]) -def test_math_function(omnisci, device, nb_version, fn_name, signature): - omnisci.reset() +def test_math_function(heavydb, device, nb_version, fn_name, signature): + heavydb.reset() - if not omnisci.has_cuda and device == 'gpu': - pytest.skip('test requires CUDA-enabled omniscidb server') + if not heavydb.has_cuda and device == 'gpu': + pytest.skip('test requires CUDA-enabled heavydb server') math_func = getattr(math, fn_name, None) if math_func is None: @@ -162,9 +163,9 @@ def test_math_function(omnisci, device, nb_version, fn_name, signature): fn.__name__ = fprefix + fn.__name__ - omnisci(signature, devices=[device])(fn) + heavydb(signature, devices=[device])(fn) - omnisci.register() + heavydb.register() if kind == 'double': assert arity <= 3, arity @@ -186,8 +187,8 @@ def test_math_function(omnisci, device, nb_version, fn_name, signature): if fn_name in ['ldexp']: xs = 'x, i' - query = f'select {xs}, {fprefix}{fn_name}({xs}) from {omnisci.table_name}' - descr, result = omnisci.sql_execute(query) + query = f'select {xs}, {fprefix}{fn_name}({xs}) from {heavydb.table_name}' + descr, result = heavydb.sql_execute(query) for args in list(result): result = args[-1] if fn_name in ['pi', 'e', 'tau', 'inf', 'nan']: @@ -320,11 +321,11 @@ def test_math_function(omnisci, device, nb_version, fn_name, signature): @pytest.mark.parametrize("device", devices) @pytest.mark.parametrize("fn_name, signature, np_func", numpy_functions, ids=["np." + item[0] for item in numpy_functions]) -def test_numpy_function(omnisci, device, nb_version, fn_name, signature, np_func): - omnisci.reset() +def test_numpy_function(heavydb, device, nb_version, fn_name, signature, np_func): + heavydb.reset() - if not omnisci.has_cuda and device == 'gpu': - pytest.skip('Test requires CUDA-enabled omniscidb server') + if not heavydb.has_cuda and device == 'gpu': + pytest.skip('Test requires CUDA-enabled heavydb server') if fn_name in ['cbrt', 'float_power']: pytest.skip(f'Numba does not support {fn_name}') @@ -364,9 +365,9 @@ def test_numpy_function(omnisci, device, nb_version, fn_name, signature, np_func if device == 'gpu' and fn_name in ['floor_divide', 'around', 'round2', 'round_']: pytest.skip(f'Missing libdevice bindigs for {fn_name}') - omnisci(signature, devices=[device])(fn) + heavydb(signature, devices=[device])(fn) - omnisci.register() + heavydb.register() if fn_name == 'ldexp': xs = ', '.join('xi') @@ -384,8 +385,8 @@ def test_numpy_function(omnisci, device, nb_version, fn_name, signature, np_func else: raise NotImplementedError(kind) - query = f'select {xs}, {fn_name}({xs}) from {omnisci.table_name}' - descr, result = omnisci.sql_execute(query) + query = f'select {xs}, {fn_name}({xs}) from {heavydb.table_name}' + descr, result = heavydb.sql_execute(query) for args in list(result): result = args[-1] expected = np_func(*args[:-1]) diff --git a/rbc/tests/test_omnisci_mlpack.py b/rbc/tests/heavyai/test_mlpack.py similarity index 62% rename from rbc/tests/test_omnisci_mlpack.py rename to rbc/tests/heavyai/test_mlpack.py index 4b829170..c7a73aa7 100644 --- a/rbc/tests/test_omnisci_mlpack.py +++ b/rbc/tests/heavyai/test_mlpack.py @@ -1,34 +1,34 @@ import pytest import re -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.errors import OmnisciServerError @pytest.fixture(scope='module') -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6)): + for o in heavydb_fixture(globals(), minimal_version=(5, 6)): yield o @pytest.mark.parametrize("func", ['dbscan', 'kmeans']) -def test_mlpack(omnisci, func): - omnisci.require_version( +def test_mlpack(heavydb, func): + heavydb.require_version( (5, 6), - 'Requires omniscidb-internal PR 5430 and omniscidb built with -DENABLE_MLPACK') + 'Requires heavydb-internal PR 5430 and heavydb built with -DENABLE_MLPACK') extra_args = dict(dbscan='cast(1 as float), 1', kmeans='1')[func] query = (f'select * from table({func}(cursor(select cast(rowid as int), f8, f8, f8 ' - f'from {omnisci.table_name}), {extra_args}, 1))') + f'from {heavydb.table_name}), {extra_args}, 1))') try: - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) except OmnisciServerError as msg: m = re.match(fr'.*Undefined function call {func!r}', msg.args[0]) if m is not None: - pytest.skip(f'test requires omniscidb server with MLPACK support: {msg}') + pytest.skip(f'test requires heavydb server with MLPACK support: {msg}') raise result = list(result) diff --git a/rbc/tests/test_omnisci_template.py b/rbc/tests/heavyai/test_template.py similarity index 60% rename from rbc/tests/test_omnisci_template.py rename to rbc/tests/heavyai/test_template.py index cd462c6f..8ed858ca 100644 --- a/rbc/tests/test_omnisci_template.py +++ b/rbc/tests/heavyai/test_template.py @@ -1,28 +1,28 @@ import os import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope="module") -def omnisci(): - for o in omnisci_fixture(globals(), minimal_version=(5, 7)): +def heavydb(): + for o in heavydb_fixture(globals(), minimal_version=(5, 7)): o.base_name = os.path.splitext(os.path.basename(__file__))[0] yield o @pytest.fixture(scope="function") -def create_columns(omnisci): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def create_columns(heavydb): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") for size in (8, 16, 32): - table_name = f"{omnisci.base_name}_{size}" + table_name = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" derived = f"derived_{size}" - omnisci.sql_execute(f"DROP TABLE IF EXISTS {table_name};") + heavydb.sql_execute(f"DROP TABLE IF EXISTS {table_name};") - omnisci.sql_execute( + heavydb.sql_execute( f""" CREATE TABLE IF NOT EXISTS {table_name} ( {base} TEXT ENCODING DICT({size}), @@ -38,29 +38,29 @@ def create_columns(omnisci): derived: ["world", "bar", "hello", "foo", "baz", "hello", "foo"], } - omnisci.load_table_columnar(table_name, **data) + heavydb.load_table_columnar(table_name, **data) - yield omnisci + yield heavydb for size in (8, 16, 32): - table_name = f"{omnisci.base_name}_{size}" - omnisci.sql_execute(f"DROP TABLE IF EXISTS {table_name}") + table_name = f"{heavydb.base_name}_{size}" + heavydb.sql_execute(f"DROP TABLE IF EXISTS {table_name}") @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (32,)) -def test_template_text(omnisci, size): +def test_template_text(heavydb, size): - if omnisci.has_cuda: - omnisci.require_version( - (5, 8), "Requires omniscidb-internal PR 5809") + if heavydb.has_cuda: + heavydb.require_version( + (5, 8), "Requires heavydb-internal PR 5809") fn = "ct_binding_template" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {base} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [ ("hello",), ("foo",), @@ -73,52 +73,52 @@ def test_template_text(omnisci, size): @pytest.mark.parametrize("col", ("i4", "f4")) -def test_template_number(omnisci, col): +def test_template_number(heavydb, col): - if omnisci.has_cuda: - omnisci.require_version( - (5, 8), "Requires omniscidb-internal PR 5809") + if heavydb.has_cuda: + heavydb.require_version( + (5, 8), "Requires heavydb-internal PR 5809") fn = "ct_binding_template" - table = omnisci.table_name + table = heavydb.table_name query = f"SELECT * FROM table({fn}(cursor(SELECT {col} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(0,), (1,), (2,), (3,), (4,)] @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) @pytest.mark.parametrize("out", (3,)) -def test_template_columnlist_text(omnisci, size, out): +def test_template_columnlist_text(heavydb, size, out): fn = "ct_binding_columnlist" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {base}, {base}, {base} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(out,)] @pytest.mark.parametrize("col,out", zip(("i4", "f4", "i2"), (1, 2, 4))) -def test_template_columnlist_number(omnisci, col, out): +def test_template_columnlist_number(heavydb, col, out): fn = "ct_binding_columnlist" - table = omnisci.table_name + table = heavydb.table_name query = f"SELECT * FROM table({fn}(cursor(SELECT {col}, {col}, {col} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(out,)] @pytest.mark.parametrize("col,out", zip(("i4", "f4"), (10, 20))) -def test_template_column_number(omnisci, col, out): - omnisci.require_version((5, 8), "Requires omniscidb-internal PR #5770") +def test_template_column_number(heavydb, col, out): + heavydb.require_version((5, 8), "Requires heavydb-internal PR #5770") fn = "ct_binding_column" - table = omnisci.table_name + table = heavydb.table_name query = f"SELECT * FROM table({fn}(cursor(SELECT {col} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(out,)] diff --git a/rbc/tests/test_omnisci_testing.py b/rbc/tests/heavyai/test_testing.py similarity index 86% rename from rbc/tests/test_omnisci_testing.py rename to rbc/tests/heavyai/test_testing.py index c08c769d..14004ffb 100644 --- a/rbc/tests/test_omnisci_testing.py +++ b/rbc/tests/heavyai/test_testing.py @@ -1,5 +1,5 @@ import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.mark.parametrize('mth', ['columnar', 'query']) @@ -9,14 +9,14 @@ def test_table_load(mth, suffix): load_columnar = mth == 'columnar' count = 0 - for omnisci in omnisci_fixture(globals(), load_columnar=load_columnar): - omnisci.require_version((5, 7, 0), - 'Requires omniscidb-internal PR 5465 [rbc PR 330]') + for heavydb in heavydb_fixture(globals(), load_columnar=load_columnar): + heavydb.require_version((5, 7, 0), + 'Requires heavydb-internal PR 5465 [rbc PR 330]') if load_columnar and suffix == 'arraynull': - omnisci.require_version((5, 7, 0), - 'Requires omniscidb-internal PR 5492 [rbc issue 245]') + heavydb.require_version((5, 7, 0), + 'Requires heavydb-internal PR 5492 [rbc issue 245]') count += 1 - descr, result = omnisci.sql_execute(f'select * from {omnisci.table_name}{suffix}') + descr, result = heavydb.sql_execute(f'select * from {heavydb.table_name}{suffix}') result = list(result) colnames = [d.name for d in descr] diff --git a/rbc/tests/test_omnisci_text.py b/rbc/tests/heavyai/test_text.py similarity index 67% rename from rbc/tests/test_omnisci_text.py rename to rbc/tests/heavyai/test_text.py index fb506dfc..8c845010 100644 --- a/rbc/tests/test_omnisci_text.py +++ b/rbc/tests/heavyai/test_text.py @@ -2,16 +2,16 @@ from collections import defaultdict import pytest -rbc_omnisci = pytest.importorskip('rbc.omniscidb') -available_version, reason = rbc_omnisci.is_available() +rbc_heavydb = pytest.importorskip('rbc.heavydb') +available_version, reason = rbc_heavydb.is_available() pytestmark = pytest.mark.skipif(not available_version, reason=reason) @pytest.fixture(scope='module') -def omnisci(): - # TODO: use omnisci_fixture from rbc/tests/__init__.py - config = rbc_omnisci.get_client_config(debug=not True) - m = rbc_omnisci.RemoteOmnisci(**config) +def heavydb(): + # TODO: use heavydb_fixture from rbc/tests/__init__.py + config = rbc_heavydb.get_client_config(debug=not True) + m = rbc_heavydb.RemoteHeavyDB(**config) table_name = os.path.splitext(os.path.basename(__file__))[0] m.sql_execute(f'DROP TABLE IF EXISTS {table_name}') @@ -47,11 +47,11 @@ def omnisci(): print('%s in deardown' % (type(msg))) -def test_table_data(omnisci): - omnisci.reset() +def test_table_data(heavydb): + heavydb.reset() - descr, result = omnisci.sql_execute( - f'select t4, t2, t1, s, n from {omnisci.table_name}') + descr, result = heavydb.sql_execute( + f'select t4, t2, t1, s, n from {heavydb.table_name}') result = list(result) assert result == [('foofoo', 'foofoo', 'fun', ['foo', 'bar'], 'fun'), @@ -61,49 +61,49 @@ def test_table_data(omnisci): ('foo', 'foo', 'foooo', ['fun', 'bar'], 'foooo')] -def test_bytes_len(omnisci): - omnisci.reset() +def test_bytes_len(heavydb): + heavydb.reset() - @omnisci('int32(Bytes, Bytes)') + @heavydb('int32(Bytes, Bytes)') def mystrlen(s, s2): return len(s) * 100 + len(s2) - sql_query_expected = f"select length(n) * 100 + length(n2) from {omnisci.table_name}" - sql_query = f"select mystrlen(n, n2) from {omnisci.table_name}" + sql_query_expected = f"select length(n) * 100 + length(n2) from {heavydb.table_name}" + sql_query = f"select mystrlen(n, n2) from {heavydb.table_name}" - descr, result_expected = omnisci.sql_execute(sql_query_expected) + descr, result_expected = heavydb.sql_execute(sql_query_expected) result_expected = list(result_expected) - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected -def test_bytes_ord(omnisci): - omnisci.reset() +def test_bytes_ord(heavydb): + heavydb.reset() - @omnisci('int64(Bytes, int32)', devices=['gpu', 'cpu']) + @heavydb('int64(Bytes, int32)', devices=['gpu', 'cpu']) def myord(s, i): return s[i] if i < len(s) else 0 - sql_query_data = f"select n from {omnisci.table_name}" - descr, data = omnisci.sql_execute(sql_query_data) + sql_query_data = f"select n from {heavydb.table_name}" + descr, data = heavydb.sql_execute(sql_query_data) data = list(data) for i in range(5): result_expected = [(ord(d[0][i]) if i < len(d[0]) else 0, ) for d in data] - sql_query = f"select myord(n, {i}) from {omnisci.table_name}" - descr, result = omnisci.sql_execute(sql_query) + sql_query = f"select myord(n, {i}) from {heavydb.table_name}" + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == result_expected -def test_bytes_return(omnisci): - omnisci.reset() +def test_bytes_return(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Bytes + from rbc.heavyai import Bytes - @omnisci('Bytes(int32, int32)') + @heavydb('Bytes(int32, int32)') def make_abc(first, n): r = Bytes(n) for i in range(n): @@ -111,18 +111,18 @@ def make_abc(first, n): return r sql_query = "select make_abc(97, 10)" - descr, result = omnisci.sql_execute(sql_query) + descr, result = heavydb.sql_execute(sql_query) result = list(result) assert result == [('abcdefghij', )] -def test_bytes_upper(omnisci): - omnisci.reset() +def test_bytes_upper(heavydb): + heavydb.reset() - from rbc.omnisci_backend import Bytes + from rbc.heavyai import Bytes - @omnisci('Bytes(Bytes)') + @heavydb('Bytes(Bytes)') def myupper(s): r = Bytes(len(s)) for i in range(len(s)): @@ -132,8 +132,8 @@ def myupper(s): r[i] = c return r - sql_query = f"select n, myupper(n) from {omnisci.table_name}" - descr, result = omnisci.sql_execute(sql_query) + sql_query = f"select n, myupper(n) from {heavydb.table_name}" + descr, result = heavydb.sql_execute(sql_query) result = list(result) for n, u in result: diff --git a/rbc/tests/test_omnisci_text_encoding.py b/rbc/tests/heavyai/test_text_encoding.py similarity index 68% rename from rbc/tests/test_omnisci_text_encoding.py rename to rbc/tests/heavyai/test_text_encoding.py index 1a9fb8ba..cf57b494 100644 --- a/rbc/tests/test_omnisci_text_encoding.py +++ b/rbc/tests/heavyai/test_text_encoding.py @@ -1,20 +1,20 @@ import os import pytest import itertools -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture @pytest.fixture(scope="module") -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals(), minimal_version=(5, 7)): + for o in heavydb_fixture(globals(), minimal_version=(5, 7)): define(o) o.base_name = os.path.splitext(os.path.basename(__file__))[0] yield o -def define(omnisci): - @omnisci("int32(Column, RowMultiplier, OutputColumn | input_id=args<0>)", +def define(heavydb): + @heavydb("int32(Column, RowMultiplier, OutputColumn | input_id=args<0>)", T=["TextEncodingDict"]) def test_shared_dict_copy(x, m, y): sz = len(x) @@ -22,7 +22,7 @@ def test_shared_dict_copy(x, m, y): y[i] = x[i] return m * sz - @omnisci("int32(Column, Column, Column, RowMultiplier, " + @heavydb("int32(Column, Column, Column, RowMultiplier, " "OutputColumn | input_id=args<1>, " "OutputColumn | input_id=args<0>, " "OutputColumn | input_id=args<2>)", @@ -35,7 +35,7 @@ def test_shared_dict_copy2(x, y, z, m, a, b, c): c[i] = z[i] return m * sz - @omnisci("int32(ColumnList, RowMultiplier, OutputColumn)", + @heavydb("int32(ColumnList, RowMultiplier, OutputColumn)", devices=['cpu']) def test_copy_column_list(lst, m, y): for j in range(len(y)): @@ -47,7 +47,7 @@ def test_copy_column_list(lst, m, y): y[j] += col[j] return lst.nrows - @omnisci("int32(ColumnList, Column, ColumnList, RowMultiplier, " + @heavydb("int32(ColumnList, Column, ColumnList, RowMultiplier, " "OutputColumn | input_id=args<0, 2>, " "OutputColumn | input_id=args<2, 0>, " "OutputColumn | input_id=args<2, 1>)", @@ -59,7 +59,7 @@ def test_copy_column_list2(lst1, col, lst2, m, a, b, c): c[j] = lst2[1][j] return lst1.nrows - @omnisci("int32(ColumnList, Column, RowMultiplier, OutputColumn | input_id=args<1>)", + @heavydb("int32(ColumnList, Column, RowMultiplier, OutputColumn | input_id=args<1>)", devices=['cpu'], T=["int32_t"], U=["TextEncodingDict"]) def test_copy_column_list3(lst, col, m, y): sz = len(col) @@ -69,19 +69,19 @@ def test_copy_column_list3(lst, col, m, y): @pytest.fixture(scope="function") -def create_columns(omnisci): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def create_columns(heavydb): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") for size in (8, 16, 32): - table_name = f"{omnisci.base_name}_{size}" + table_name = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" derived = f"derived_{size}" another = f"another_{size}" - omnisci.sql_execute(f"DROP TABLE IF EXISTS {table_name};") + heavydb.sql_execute(f"DROP TABLE IF EXISTS {table_name};") - omnisci.sql_execute( + heavydb.sql_execute( f""" CREATE TABLE IF NOT EXISTS {table_name} ( {base} TEXT ENCODING DICT({size}), @@ -99,26 +99,26 @@ def create_columns(omnisci): derived: ["world", "bar", "hello", "foo", "baz", "hello", "foo"], } - omnisci.load_table_columnar(table_name, **data) + heavydb.load_table_columnar(table_name, **data) - yield omnisci + yield heavydb for size in (8, 16, 32): - table_name = f"{omnisci.base_name}_{size}" - omnisci.sql_execute(f"DROP TABLE IF EXISTS {table_name}") + table_name = f"{heavydb.base_name}_{size}" + heavydb.sql_execute(f"DROP TABLE IF EXISTS {table_name}") @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32)) -def test_text_encoding_shared_dict(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_text_encoding_shared_dict(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = "test_shared_dict_copy" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {base} FROM {table}), 1))" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [ ("hello",), @@ -133,24 +133,24 @@ def test_text_encoding_shared_dict(omnisci, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) -def test_text_encoding_shared_dict2(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5719") +def test_text_encoding_shared_dict2(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5719") fn = "test_shared_dict_copy2" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" another = f"another_{size}" ans_query = (f"SELECT {other}, {base}, {another} FROM {table};") - _, ans = omnisci.sql_execute(ans_query) + _, ans = heavydb.sql_execute(ans_query) query = (f"SELECT * FROM table({fn}(" f"cursor(SELECT {base} FROM {table}), " f"cursor(SELECT {other} FROM {table}), " f"cursor(SELECT {another} FROM {table}), " "1))") - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(ans) == list(result) @@ -158,18 +158,18 @@ def test_text_encoding_shared_dict2(omnisci, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("col_name", ("base", "other", "another")) @pytest.mark.parametrize("size", (8, 16, 32)) -def test_text_encoding_shared_dict3(omnisci, col_name, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_text_encoding_shared_dict3(heavydb, col_name, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = "test_shared_dict_copy" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" col = f"{col_name}_{size}" ans_query = (f"SELECT {col} FROM {table};") - _, ans = omnisci.sql_execute(ans_query) + _, ans = heavydb.sql_execute(ans_query) query = f"SELECT * FROM table({fn}(cursor(SELECT {col} FROM {table}), 1))" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == list(ans) @@ -177,22 +177,22 @@ def test_text_encoding_shared_dict3(omnisci, col_name, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) @pytest.mark.parametrize("num_column_list", (1, 2, 3)) -def test_text_encoding_column_list(omnisci, size, num_column_list): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_text_encoding_column_list(heavydb, size, num_column_list): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = "test_copy_column_list" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" cols = ', '.join([base] * num_column_list) - _, expected = omnisci.sql_execute( + _, expected = heavydb.sql_execute( f"SELECT * FROM table({fn}(cursor(SELECT {base} FROM {table}), 1));") query = ( f"SELECT * FROM table({fn}(" f"cursor(SELECT {cols} FROM {table}), " "1));") - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(map(lambda x: x[0] * num_column_list, expected)) == \ list(itertools.chain.from_iterable(result)) @@ -200,26 +200,26 @@ def test_text_encoding_column_list(omnisci, size, num_column_list): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (32,)) -def test_text_encoding_column_list2(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5719") +def test_text_encoding_column_list2(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5719") fn = "test_copy_column_list2" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" another = f"another_{size}" cols = f"{base}, {other}, {another}" ans_query = (f"SELECT {another}, {base}, {other} FROM {table};") - _, ans = omnisci.sql_execute(ans_query) + _, ans = heavydb.sql_execute(ans_query) query = ( f"SELECT * FROM table({fn}(" f"cursor(SELECT {cols} FROM {table}), " - f"cursor(SELECT i4 FROM {omnisci.table_name}), " + f"cursor(SELECT i4 FROM {heavydb.table_name}), " f"cursor(SELECT {cols} FROM {table}), " "1));") - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(ans) == list(result) @@ -227,54 +227,54 @@ def test_text_encoding_column_list2(omnisci, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (32,)) @pytest.mark.parametrize("num_cols", (1, 2, 3, 4)) -def test_text_encoding_column_list3(omnisci, size, num_cols): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5719") +def test_text_encoding_column_list3(heavydb, size, num_cols): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5719") fn = "test_copy_column_list3" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" cols = ", ".join(["i4"] * num_cols) - _, ans = omnisci.sql_execute(f"SELECT {base} FROM {table} LIMIT 5;") + _, ans = heavydb.sql_execute(f"SELECT {base} FROM {table} LIMIT 5;") query = ( f"SELECT * FROM table({fn}(" - f"cursor(SELECT {cols} FROM {omnisci.table_name}), " + f"cursor(SELECT {cols} FROM {heavydb.table_name}), " f"cursor(SELECT {base} FROM {table}), " "1));") - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(ans) == list(result) @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) -def test_text_encoding_count(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_text_encoding_count(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = "test_shared_dict_copy" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = f"SELECT COUNT(out0) FROM table({fn}(cursor(SELECT {base} FROM {table}), 1))" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [(7,)] @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) -def test_text_encoding_order_by(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_text_encoding_order_by(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = "test_shared_dict_copy" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = ( f"SELECT out0 FROM table({fn}(cursor(SELECT {base} FROM {table}), 1))" " ORDER BY out0;") - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [ ('bar',), @@ -289,15 +289,15 @@ def test_text_encoding_order_by(omnisci, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) -def test_ct_binding_dict_encoded1(omnisci, size): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5492") +def test_ct_binding_dict_encoded1(heavydb, size): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5492") fn = 'ct_binding_dict_encoded1' - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {base} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) assert list(result) == [ ("hello",), ("foo",), @@ -312,16 +312,16 @@ def test_ct_binding_dict_encoded1(omnisci, size): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) @pytest.mark.parametrize("fn_suffix", ("2", "3",)) -def test_ct_binding_dict_encoded23(omnisci, size, fn_suffix): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5719") +def test_ct_binding_dict_encoded23(heavydb, size, fn_suffix): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5719") fn = f"ct_binding_dict_encoded{fn_suffix}" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {base}, {other} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) out0 = ["hello", "foo", "foofoo", "world", "bar", "foo", "foofoo"] out1 = ["a1", "b2", "c3", "d4", "e5", "f6", "g7"] @@ -335,16 +335,16 @@ def test_ct_binding_dict_encoded23(omnisci, size, fn_suffix): @pytest.mark.usefixtures("create_columns") @pytest.mark.parametrize("size", (8, 16, 32,)) @pytest.mark.parametrize("fn_suffix", ("4", "5", "6")) -def test_ct_binding_dict_encoded45(omnisci, size, fn_suffix): - omnisci.require_version((5, 7), "Requires omniscidb-internal PR 5719") +def test_ct_binding_dict_encoded45(heavydb, size, fn_suffix): + heavydb.require_version((5, 7), "Requires heavydb-internal PR 5719") fn = f"ct_binding_dict_encoded{fn_suffix}" - table = f"{omnisci.base_name}_{size}" + table = f"{heavydb.base_name}_{size}" base = f"base_{size}" other = f"other_{size}" query = f"SELECT * FROM table({fn}(cursor(SELECT {other}, {base}, {other} FROM {table})));" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) out4 = ["a1", "b2", "c3", "d4", "e5", "f6", "g7"] out5 = ["hello", "foo", "foofoo", "world", "bar", "foo", "foofoo"] diff --git a/rbc/tests/test_omnisci_treelite.py b/rbc/tests/heavyai/test_treelite.py similarity index 75% rename from rbc/tests/test_omnisci_treelite.py rename to rbc/tests/heavyai/test_treelite.py index f97528d9..4329aad6 100644 --- a/rbc/tests/test_omnisci_treelite.py +++ b/rbc/tests/heavyai/test_treelite.py @@ -1,21 +1,21 @@ import os import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.external import external @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals(), minimal_version=(5, 6), debug=not True): +def heavydb(): + for o in heavydb_fixture(globals(), minimal_version=(5, 6), debug=not True): yield o @pytest.fixture(scope='module') -def boston_house_prices(omnisci): +def boston_house_prices(heavydb): # Upload Boston house prices to server, notice that we collect all # row values expect the last one (MEDV) to a FLOAT array. - csv_file = os.path.join(os.path.dirname(__file__), 'boston_house_prices.csv') + csv_file = os.path.join(os.path.dirname(__file__), '..', 'boston_house_prices.csv') data0 = [] medv0 = [] for i, line in enumerate(open(csv_file).readlines()): @@ -27,19 +27,19 @@ def boston_house_prices(omnisci): assert len(row) == len(header) data0.append(row[:-1]) medv0.append(row[-1]) - table_name = f'{omnisci.table_name}bhp' + table_name = f'{heavydb.table_name}bhp' - omnisci.sql_execute(f'DROP TABLE IF EXISTS {table_name}') - omnisci.sql_execute(f'CREATE TABLE IF NOT EXISTS {table_name} (data FLOAT[], medv FLOAT);') - omnisci.load_table_columnar(table_name, data=data0, medv=medv0) + heavydb.sql_execute(f'DROP TABLE IF EXISTS {table_name}') + heavydb.sql_execute(f'CREATE TABLE IF NOT EXISTS {table_name} (data FLOAT[], medv FLOAT);') + heavydb.load_table_columnar(table_name, data=data0, medv=medv0) - yield omnisci + yield heavydb - omnisci.sql_execute(f'DROP TABLE IF EXISTS {table_name}') + heavydb.sql_execute(f'DROP TABLE IF EXISTS {table_name}') -def test_boston_house_prices(omnisci, boston_house_prices): - if omnisci.compiler is None: +def test_boston_house_prices(heavydb, boston_house_prices): + if heavydb.compiler is None: pytest.skip('test requires C/C++ to LLVM IR compiler') treelite = pytest.importorskip("treelite") @@ -50,9 +50,9 @@ def test_boston_house_prices(omnisci, boston_house_prices): import tempfile # Get training data from server: - table_name = f'{omnisci.table_name}bhp' + table_name = f'{heavydb.table_name}bhp' - descr, result = omnisci.sql_execute( + descr, result = heavydb.sql_execute( f'SELECT rowid, data, medv FROM {table_name} ORDER BY rowid LIMIT 50') result = list(result) medv = np.array([medv for _, data, medv in result]) @@ -91,11 +91,11 @@ def test_boston_house_prices(omnisci, boston_house_prices): # be now called from a UDF. # Define predict function as UDF. Notice that the xgboost null - # values are different from omniscidb null values, so we'll remap + # values are different from heavydb null values, so we'll remap # before calling predict_float: null_value = np.int32(-1).view(np.float32) - @omnisci('float(float[], int32)', devices=[device]) + @heavydb('float(float[], int32)', devices=[device]) def predict(data, pred_margin): for i in range(len(data)): if data.is_null(i): @@ -105,13 +105,13 @@ def predict(data, pred_margin): # Compile C model to LLVM IR. In future, we might want this # compilation to happen in the server side as the client might not # have clang compiler installed. - model_llvmir = omnisci.compiler(model_c, flags=['-I' + working_dir]) + model_llvmir = heavydb.compiler(model_c, flags=['-I' + working_dir]) # RBC will link_in the LLVM IR module - omnisci.user_defined_llvm_ir[device] = model_llvmir + heavydb.user_defined_llvm_ir[device] = model_llvmir # Call predict on data in the server: - descr, result = omnisci.sql_execute('SELECT rowid, predict(data, 2) FROM' + descr, result = heavydb.sql_execute('SELECT rowid, predict(data, 2) FROM' f' {table_name} ORDER BY rowid') result = list(result) predict_medv = np.array([r[1] for r in result]) diff --git a/rbc/tests/test_omnisci_udtf.py b/rbc/tests/heavyai/test_udtf.py similarity index 76% rename from rbc/tests/test_omnisci_udtf.py rename to rbc/tests/heavyai/test_udtf.py index 2be75890..c26e6613 100644 --- a/rbc/tests/test_omnisci_udtf.py +++ b/rbc/tests/heavyai/test_udtf.py @@ -1,22 +1,22 @@ import pytest from rbc.errors import NumbaTypeError -from rbc.tests import omnisci_fixture, sql_execute -from rbc.externals.omniscidb import table_function_error +from rbc.tests import heavydb_fixture, sql_execute +from rbc.externals.heavydb import table_function_error import numpy as np @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): define(o) yield o -def define(omnisci): +def define(heavydb): T = ['int64', 'int32', 'int16', 'int8', 'float64', 'float32'] - @omnisci('int32(Cursor, T, RowMultiplier,' + @heavydb('int32(Cursor, T, RowMultiplier,' ' OutputColumn, OutputColumn)', T=T) def sqlmultiply(rowid, col, alpha, row_multiplier, rowid_out, out): for i in range(len(col)): @@ -25,7 +25,7 @@ def sqlmultiply(rowid, col, alpha, row_multiplier, rowid_out, out): rowid_out[j] = i return len(col) - @omnisci('int32(Cursor, T, RowMultiplier,' + @heavydb('int32(Cursor, T, RowMultiplier,' ' OutputColumn, OutputColumn)', T=T) def sqladd(rowid, col, alpha, row_multiplier, rowid_out, out): for i in range(len(col)): @@ -34,7 +34,7 @@ def sqladd(rowid, col, alpha, row_multiplier, rowid_out, out): rowid_out[j] = i return len(col) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)', T=T) def sqladd2(rowid1, col1, rowid2, col2, row_multiplier, rowid_out, out): for i1 in range(len(col1)): @@ -47,7 +47,7 @@ def sqladd2(rowid1, col1, rowid2, col2, row_multiplier, rowid_out, out): break return len(col1) - @omnisci('int32(Cursor, Cursor, RowMultiplier,' + @heavydb('int32(Cursor, Cursor, RowMultiplier,' ' OutputColumn, OutputColumn)', T=T) def sqlmultiply2(rowid1, col1, rowid2, col2, row_multiplier, rowid_out, out): for i1 in range(len(col1)): @@ -62,11 +62,11 @@ def sqlmultiply2(rowid1, col1, rowid2, col2, row_multiplier, rowid_out, out): @pytest.mark.parametrize("kind", ['i8', 'i4', 'i2', 'i1', 'f8', 'f4']) -def test_composition(omnisci, kind): - omnisci.require_version((5, 6), 'Requires omniscidb-internal PR 5440') +def test_composition(heavydb, kind): + heavydb.require_version((5, 6), 'Requires heavydb-internal PR 5440') def tonp(query): - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) return tuple(np.array(a) for a in zip(*list(result))) sqltyp = dict(i8='BIGINT', i4='INT', i2='SMALLINT', i1='TINYINT', @@ -110,7 +110,7 @@ def isok(self): def __repr__(self): return f'{type(self).__name__}({self.arr!r}, {self.select!r})' - select0 = f'select rowid, {kind} from {omnisci.table_name}' + select0 = f'select rowid, {kind} from {heavydb.table_name}' arr0 = tonp(select0)[1] a = Algebra(arr0, select0) @@ -130,14 +130,14 @@ def __repr__(self): assert ((a + 2) * a + 3).isok() -def test_table_function_manager(omnisci): - omnisci.require_version((5, 9), 'Requires omniscidb-internal PR 6035') +def test_table_function_manager(heavydb): + heavydb.require_version((5, 9), 'Requires heavydb-internal PR 6035') - @omnisci('int32(TableFunctionManager, Column, OutputColumn)') + @heavydb('int32(TableFunctionManager, Column, OutputColumn)') def my_manager_error(mgr, col, out): return mgr.error_message("TableFunctionManager error_message!") - @omnisci('int32(TableFunctionManager, Column, OutputColumn)') + @heavydb('int32(TableFunctionManager, Column, OutputColumn)') def my_manager_row_size(mgr, col, out): size = len(col) mgr.set_output_row_size(size) @@ -145,16 +145,16 @@ def my_manager_row_size(mgr, col, out): out[i] = col[i] return size - with pytest.raises(omnisci.thrift_client.thrift.TMapDException) as exc: - omnisci.sql_execute( + with pytest.raises(heavydb.thrift_client.thrift.TMapDException) as exc: + heavydb.sql_execute( f'select out0 from table(my_manager_error(' - f'cursor(select f8 from {omnisci.table_name})));') + f'cursor(select f8 from {heavydb.table_name})));') assert exc.match('Error executing table function: TableFunctionManager error_message!') - _, result = omnisci.sql_execute( + _, result = heavydb.sql_execute( f'select out0 from table(my_manager_row_size(' - f'cursor(select f8 from {omnisci.table_name})));') + f'cursor(select f8 from {heavydb.table_name})));') expected = [(0.0,), (1.0,), (2.0,), (3.0,), (4.0,)] assert(list(result) == expected) @@ -162,14 +162,14 @@ def my_manager_row_size(mgr, col, out): @pytest.mark.parametrize("sleep", ['ct_sleep1', 'ct_sleep2']) @pytest.mark.parametrize("mode", [0, 1, 2, 3, 4]) -def test_parallel_execution(omnisci, sleep, mode): - """This test is affected by omniscidb server option --num-executors. +def test_parallel_execution(heavydb, sleep, mode): + """This test is affected by heavydb server option --num-executors. Here we start a number of identical tasks at the same time and observe if the tasks are run in-parallel (ct_sleep2) or not (ct_sleep1). """ - omnisci.require_version((5, 9), 'Requires omniscidb-internal PR 5901') + heavydb.require_version((5, 9), 'Requires heavydb-internal PR 5901') from multiprocessing import Process, Array def func(seconds, mode, a): @@ -194,9 +194,9 @@ def func(seconds, mode, a): a[i] = v[0] # mode > 1 correspond to exception tests. Increasing nof_jobs for - # such tests and rerunning pytest many times may lead to omniscidb + # such tests and rerunning pytest many times may lead to heavydb # server crash if num-executors>1. This may indicate that - # omniscidb error handling is not completely thread-save. + # heavydb error handling is not completely thread-save. nof_jobs = 5 if mode <= 1 else 1 sleep_seconds = 1 @@ -230,11 +230,11 @@ def func(seconds, mode, a): + '*' * ((end - start) // 100)) -def test_table_function_error(omnisci): - omnisci.require_version((5, 8), 'Requires omniscidb-internal PR 5879') - omnisci.reset() +def test_table_function_error(heavydb): + heavydb.require_version((5, 8), 'Requires heavydb-internal PR 5879') + heavydb.reset() - @omnisci('int32(Column, double, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, double, RowMultiplier, OutputColumn)') def my_divide(column, k, row_multiplier, out): if k == 0: return table_function_error('division by zero') @@ -242,30 +242,30 @@ def my_divide(column, k, row_multiplier, out): out[i] = column[i] / k return len(column) - descr, result = omnisci.sql_execute(f""" + descr, result = heavydb.sql_execute(f""" select * from table( - my_divide(CURSOR(SELECT f8 FROM {omnisci.table_name}), 2) + my_divide(CURSOR(SELECT f8 FROM {heavydb.table_name}), 2) ); """) assert list(result) == [(0.0,), (0.5,), (1.0,), (1.5,), (2.0,)] - with pytest.raises(omnisci.thrift_client.thrift.TMapDException) as exc: - omnisci.sql_execute(f""" + with pytest.raises(heavydb.thrift_client.thrift.TMapDException) as exc: + heavydb.sql_execute(f""" select * from table( - my_divide(CURSOR(SELECT f8 FROM {omnisci.table_name}), 0) + my_divide(CURSOR(SELECT f8 FROM {heavydb.table_name}), 0) ); """) assert exc.match('Error executing table function: division by zero') -def test_raise_error(omnisci): - omnisci.require_version((5, 8), 'Requires omniscidb-internal PR 5879') - omnisci.reset() +def test_raise_error(heavydb): + heavydb.require_version((5, 8), 'Requires heavydb-internal PR 5879') + heavydb.reset() with pytest.raises(NumbaTypeError) as exc: - @omnisci('int32(Column, double, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, double, RowMultiplier, OutputColumn)') def my_divide(column, k, row_multiplier, out): if k == 0: raise ValueError('division by zero') @@ -273,21 +273,21 @@ def my_divide(column, k, row_multiplier, out): out[i] = column[i] / k return len(column) - omnisci.register() + heavydb.register() assert exc.match('raise statement is not supported') - omnisci.reset() + heavydb.reset() -def test_issue_235(omnisci): +def test_issue_235(heavydb): - @omnisci('int32(Column, RowMultiplier, OutputColumn)') + @heavydb('int32(Column, RowMultiplier, OutputColumn)') def text_rbc_copy_rowmul(x, m, y): for i in range(len(x)): y[i] = x[i] return len(x) - @omnisci('int32(Cursor, RowMultiplier, OutputColumn, OutputColumn)') # noqa: E501 + @heavydb('int32(Cursor, RowMultiplier, OutputColumn, OutputColumn)') # noqa: E501 def text_rbc_copy_rowmul(x, x2, m, y, y2): # noqa: F811 for i in range(len(x)): y[i] = x[i] @@ -295,6 +295,6 @@ def text_rbc_copy_rowmul(x, x2, m, y, y2): # noqa: F811 return len(x) query = (f'select * from table(text_rbc_copy_rowmul(' - f'cursor(select i8, f8 from {omnisci.table_name}), 1));') - _, result = omnisci.sql_execute(query) + f'cursor(select i8, f8 from {heavydb.table_name}), 1));') + _, result = heavydb.sql_execute(query) assert list(result) == list(zip(np.arange(5), np.arange(5, dtype=np.float64))) diff --git a/rbc/tests/stdlib/test_array_api.py b/rbc/tests/stdlib/test_array_api.py index 58180e03..8e8e66f1 100644 --- a/rbc/tests/stdlib/test_array_api.py +++ b/rbc/tests/stdlib/test_array_api.py @@ -1,5 +1,5 @@ from rbc.stdlib import array_api as xp -from rbc.omnisci_backend.omnisci_buffer import free_buffer +from rbc.heavyai.buffer import free_buffer from ..test_rbclib import djit # noqa: F401 diff --git a/rbc/tests/test_ci.py b/rbc/tests/test_ci.py index 07b6d45d..b403a5bb 100644 --- a/rbc/tests/test_ci.py +++ b/rbc/tests/test_ci.py @@ -1,13 +1,13 @@ import os import sys import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.utils import parse_version @pytest.fixture(scope='module') -def omnisci(): - for o in omnisci_fixture(globals()): +def heavydb(): + for o in heavydb_fixture(globals()): yield o @@ -26,20 +26,20 @@ def test_python_version(): assert expected == current_stripped -def test_omniscidb_version(omnisci): +def test_heavydb_version(heavydb): varname = 'EXPECTED_OMNISCIDB_VERSION' - current = omnisci.version + current = heavydb.version expected = os.environ.get(varname) if 'CI' in os.environ and expected is None: pytest.fail("OmniSciDB server is not running") if expected is None: pytest.skip( f'Undefined environment variable {varname},' - f' cannot test omniscidb version (current={".".join(map(str, current))})') + f' cannot test heavydb version (current={".".join(map(str, current))})') if expected == 'dev': assert current[:2] >= (5, 8), current # TODO: update dev version periodically pytest.skip( - f'omniscidb dev version is {".".join(map(str, current))}') + f'heavydb dev version is {".".join(map(str, current))}') expected = parse_version(expected) current_stripped = current[:len(expected)] assert expected == current_stripped diff --git a/rbc/tests/test_externals_cmath.py b/rbc/tests/test_externals_cmath.py index 84b4829a..a27a3183 100644 --- a/rbc/tests/test_externals_cmath.py +++ b/rbc/tests/test_externals_cmath.py @@ -1,7 +1,7 @@ import sys import atexit import pytest -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.remotejit import RemoteJIT from rbc.externals import cmath from rbc.typesystem import Type @@ -29,9 +29,9 @@ def ljit(request): @pytest.fixture(scope="module") -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals()): + for o in heavydb_fixture(globals()): define(o) yield o @@ -181,18 +181,18 @@ def _get_pyfunc(fname): @pytest.mark.parametrize("fname,sig", cmath_funcs, ids=[item[0] for item in cmath_funcs]) -def test_external_cmath_omnisci(omnisci, fname, sig): +def test_external_cmath_heavydb(heavydb, fname, sig): if fname in ["logb", "ilogb", "modf"]: pytest.skip(f"cmath function {fname} not supported") if fname in ["frexp", "nan"]: - pytest.xfail(f"cmath function {fname} crashes omniscidb server") + pytest.xfail(f"cmath function {fname} crashes heavydb server") if fname in ["remainder"]: pytest.xfail(f"cmath.{fname} wrong output!") - table = omnisci.table_name + table = heavydb.table_name query_func = _get_query_func(fname) pyfunc = _get_pyfunc(fname) @@ -225,7 +225,7 @@ def test_external_cmath_omnisci(omnisci, fname, sig): else: query = f"SELECT f8+10.0, {query_func}(f8+10.0) from {table}" - _, result = omnisci.sql_execute(query) + _, result = heavydb.sql_execute(query) for values in result: if len(values) == 3: diff --git a/rbc/tests/test_externals_libdevice.py b/rbc/tests/test_externals_libdevice.py index e36ebae8..100530af 100644 --- a/rbc/tests/test_externals_libdevice.py +++ b/rbc/tests/test_externals_libdevice.py @@ -1,7 +1,7 @@ import pytest from functools import reduce from typing import Tuple -from rbc.tests import omnisci_fixture +from rbc.tests import heavydb_fixture from rbc.externals import libdevice libdevicefuncs = pytest.importorskip("numba.cuda.libdevicefuncs") @@ -16,21 +16,21 @@ @pytest.fixture(scope="module") -def omnisci(): +def heavydb(): - for o in omnisci_fixture(globals()): + for o in heavydb_fixture(globals()): if not o.has_cuda: pytest.skip("cuda is not enabled") define(o) yield o -def define(omnisci): +def define(heavydb): def inner(fname: str, retty: str, argtypes: Tuple[str]): cmath_fn = getattr(libdevice, fname) arity = len(argtypes) - # define omnisci callable + # define heavydb callable if arity == 1: def fn(a): @@ -46,8 +46,8 @@ def fn(a, b): def fn(a, b, c): return cmath_fn(a, b, c) - fn.__name__ = f"{omnisci.table_name}_{fname[5:]}" - fn = omnisci(f"{retty}({', '.join(argtypes)})", devices=["gpu"])(fn) + fn.__name__ = f"{heavydb.table_name}_{fname[5:]}" + fn = heavydb(f"{retty}({', '.join(argtypes)})", devices=["gpu"])(fn) for fname, retty, argtys, has_ptr_arg in funcs: if has_ptr_arg: @@ -68,12 +68,12 @@ def fn(a, b, c): @pytest.mark.parametrize( "fname,retty,argtys,has_ptr_arg", funcs, ids=[item[0] for item in funcs] ) -def test_externals_libdevice(omnisci, fname, retty, argtys, has_ptr_arg): +def test_externals_libdevice(heavydb, fname, retty, argtys, has_ptr_arg): if has_ptr_arg: pytest.skip(f"{fname} has a pointer argument") - func_name = f"{omnisci.table_name}_{fname[5:]}" - table = f"{omnisci.table_name}" + func_name = f"{heavydb.table_name}_{fname[5:]}" + table = f"{heavydb.table_name}" if fname[5:] in ["brev", "brevll"]: query = f"SELECT {func_name}(i4+4) FROM {table}" @@ -83,4 +83,4 @@ def test_externals_libdevice(omnisci, fname, retty, argtys, has_ptr_arg): cols = ", ".join(tuple(map(lambda x: cols_dict[x], argtys))) query = f"SELECT {func_name}({cols}) FROM {table}" - _, _ = omnisci.sql_execute(query) + _, _ = heavydb.sql_execute(query)