From e226f23e77407378f31fc745a1718f99318b2081 Mon Sep 17 00:00:00 2001 From: Connor Boyle Date: Fri, 8 May 2020 16:33:27 -0600 Subject: [PATCH 1/4] Update for TinyDB v4 --- tinymongo/tinymongo.py | 225 +++++++++++++++++++++-------------------- 1 file changed, 115 insertions(+), 110 deletions(-) diff --git a/tinymongo/tinymongo.py b/tinymongo/tinymongo.py index 4c80d7d..1f69e01 100644 --- a/tinymongo/tinymongo.py +++ b/tinymongo/tinymongo.py @@ -12,36 +12,34 @@ from uuid import uuid1 from tinydb import Query, TinyDB, where -from .results import ( - InsertOneResult, - InsertManyResult, - UpdateResult, - DeleteResult -) +from .results import InsertOneResult, InsertManyResult, UpdateResult, DeleteResult from .errors import DuplicateKeyError try: - basestring + basestring except NameError: - basestring = str + basestring = str logger = logging.getLogger(__name__) def Q(query, key): - return reduce(lambda partial_query, field: partial_query[field], key.split('.'), query) + return reduce( + lambda partial_query, field: partial_query[field], key.split("."), query + ) class TinyMongoClient(object): """Represents the Tiny `db` client""" + def __init__(self, foldername=u"tinydb", **kwargs): """Initialize container folder""" self._foldername = foldername try: os.mkdir(foldername) except OSError as x: - logger.info('{}'.format(x)) + logger.info("{}".format(x)) @property def _storage(self): @@ -68,7 +66,7 @@ def _storage(self): return serialization """ - return TinyDB.DEFAULT_STORAGE + return TinyDB.default_storage_class def __getitem__(self, key): """Gets a new or existing database based in key""" @@ -85,13 +83,13 @@ def __getattr__(self, name): class TinyMongoDatabase(object): """Representation of a Pymongo database""" + def __init__(self, database, foldername, storage): """Initialize a TinyDB file named as the db name in the given folder """ self._foldername = foldername self.tinydb = TinyDB( - os.path.join(foldername, database + u".json"), - storage=storage + os.path.join(foldername, database + u".json"), storage=storage ) def __getattr__(self, name): @@ -162,8 +160,10 @@ def drop(self, **kwargs): exist. """ if self.table: - self.parent.tinydb.purge_table(self.tablename) + self.parent.tinydb.drop_table(self.tablename) return True + # from tinydb.database import TinyDB + # TinyDB(). else: return False @@ -187,19 +187,19 @@ def insert_one(self, doc, *args, **kwargs): if not isinstance(doc, dict): raise ValueError(u'"doc" must be a dict') - _id = doc[u'_id'] = doc.get('_id') or generate_id() + _id = doc[u"_id"] = doc.get("_id") or generate_id() - bypass_document_validation = kwargs.get('bypass_document_validation') + bypass_document_validation = kwargs.get("bypass_document_validation") if bypass_document_validation is True: # insert doc without validation of duplicated `_id` eid = self.table.insert(doc) else: - existing = self.find_one({'_id': _id}) + existing = self.find_one({"_id": _id}) if existing is None: eid = self.table.insert(doc) else: raise DuplicateKeyError( - u'_id:{0} already exists in collection:{1}'.format( + u"_id:{0} already exists in collection:{1}".format( _id, self.tablename ) ) @@ -218,21 +218,21 @@ def insert_many(self, docs, *args, **kwargs): if not isinstance(docs, list): raise ValueError(u'"insert_many" requires a list input') - bypass_document_validation = kwargs.get('bypass_document_validation') + bypass_document_validation = kwargs.get("bypass_document_validation") if bypass_document_validation is not True: # get all _id in once, to reduce I/O. (without projection) - existing = [doc['_id'] for doc in self.find({})] + existing = [doc["_id"] for doc in self.find({})] _ids = list() for doc in docs: - _id = doc[u'_id'] = doc.get('_id') or generate_id() + _id = doc[u"_id"] = doc.get("_id") or generate_id() if bypass_document_validation is not True: if _id in existing: raise DuplicateKeyError( - u'_id:{0} already exists in collection:{1}'.format( + u"_id:{0} already exists in collection:{1}".format( _id, self.tablename ) ) @@ -244,7 +244,7 @@ def insert_many(self, docs, *args, **kwargs): return InsertManyResult( eids=[eid for eid in results], - inserted_ids=[inserted_id for inserted_id in _ids] + inserted_ids=[inserted_id for inserted_id in _ids], ) def parse_query(self, query): @@ -255,11 +255,11 @@ def parse_query(self, query): query :return: composite Query() """ - logger.debug(u'query to parse2: {}'.format(query)) + logger.debug(u"query to parse2: {}".format(query)) # this should find all records if query == {} or query is None: - return Query()._id != u'-1' # noqa + return Query()._id != u"-1" # noqa q = None # find the final result of the generator @@ -269,7 +269,7 @@ def parse_query(self, query): else: q = q & c - logger.debug(u'new query item2: {}'.format(q)) + logger.debug(u"new query item2: {}".format(q)) return q @@ -284,7 +284,7 @@ def parse_condition(self, query, prev_key=None, last_prev_key=None): Query() object containing all conditions """ # use this to determine gt/lt/eq on prev_query - logger.debug(u'query: {} prev_query: {}'.format(query, prev_key)) + logger.debug(u"query: {} prev_query: {}".format(query, prev_key)) q = Query() conditions = None @@ -297,75 +297,91 @@ def parse_condition(self, query, prev_key=None, last_prev_key=None): # deal with the conditions for key, value in query.items(): - logger.debug(u'conditions: {} {}'.format(key, value)) + logger.debug(u"conditions: {} {}".format(key, value)) - if key == u'$gte': + if key == u"$gte": conditions = ( - Q(q, prev_key) >= value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) >= value)) if prev_key != "$not" \ - else (q[last_prev_key] < value) - elif key == u'$gt': + (Q(q, prev_key) >= value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) >= value)) + if prev_key != "$not" + else (q[last_prev_key] < value) + ) + elif key == u"$gt": conditions = ( - Q(q, prev_key) > value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) > value)) if prev_key != "$not" \ - else (q[last_prev_key] <= value) - elif key == u'$lte': + (Q(q, prev_key) > value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) > value)) + if prev_key != "$not" + else (q[last_prev_key] <= value) + ) + elif key == u"$lte": conditions = ( - Q(q, prev_key) <= value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) <= value)) if prev_key != "$not" \ - else (q[last_prev_key] > value) - elif key == u'$lt': + (Q(q, prev_key) <= value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) <= value)) + if prev_key != "$not" + else (q[last_prev_key] > value) + ) + elif key == u"$lt": conditions = ( - Q(q, prev_key) < value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) < value)) if prev_key != "$not" \ - else (q[last_prev_key] >= value) - elif key == u'$ne': + (Q(q, prev_key) < value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) < value)) + if prev_key != "$not" + else (q[last_prev_key] >= value) + ) + elif key == u"$ne": conditions = ( - Q(q, prev_key) != value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) != value))if prev_key != "$not" \ - else (q[last_prev_key] == value) - elif key == u'$not': + (Q(q, prev_key) != value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) != value)) + if prev_key != "$not" + else (q[last_prev_key] == value) + ) + elif key == u"$not": if not isinstance(value, dict) and not isinstance(value, list): conditions = ( - Q(q, prev_key) != value - ) if not conditions and prev_key != "$not" \ - else (conditions & (Q(q, prev_key) != value)) \ - if prev_key != "$not" else (q[last_prev_key] >= value) + (Q(q, prev_key) != value) + if not conditions and prev_key != "$not" + else (conditions & (Q(q, prev_key) != value)) + if prev_key != "$not" + else (q[last_prev_key] >= value) + ) else: # let the value's condition be parsed below pass - elif key == u'$regex': - value = value.replace('\\\\\\', '|||') - value = value.replace('\\\\', '|||') - regex = value.replace('\\', '') - regex = regex.replace('|||', '\\') - currCond = (where(prev_key).matches(regex)) + elif key == u"$regex": + value = value.replace("\\\\\\", "|||") + value = value.replace("\\\\", "|||") + regex = value.replace("\\", "") + regex = regex.replace("|||", "\\") + currCond = where(prev_key).matches(regex) conditions = currCond if not conditions else (conditions & currCond) - elif key in ['$and', '$or', '$in', '$all']: + elif key in ["$and", "$or", "$in", "$all"]: pass else: - # don't want to use the previous key if this is a secondary key # (fixes multiple item query that includes $ codes) if not isinstance(value, dict) and not isinstance(value, list): conditions = ( - (Q(q, key) == value) | (Q(q, key).any([value])) - ) if not conditions else (conditions & ((Q(q, key) == value) | (Q(q, key).any([value])))) + ((Q(q, key) == value) | (Q(q, key).any([value]))) + if not conditions + else ( + conditions + & ((Q(q, key) == value) | (Q(q, key).any([value]))) + ) + ) prev_key = key - logger.debug(u'c: {}'.format(conditions)) + logger.debug(u"c: {}".format(conditions)) if isinstance(value, dict): # yield from self.parse_condition(value, key) for parse_condition in self.parse_condition(value, key, prev_key): yield parse_condition elif isinstance(value, list): - if key == '$and': + if key == "$and": grouped_conditions = None for spec in value: for parse_condition in self.parse_condition(spec): @@ -375,7 +391,7 @@ def parse_condition(self, query, prev_key=None, last_prev_key=None): else grouped_conditions & parse_condition ) yield grouped_conditions - elif key == '$or': + elif key == "$or": grouped_conditions = None for spec in value: for parse_condition in self.parse_condition(spec): @@ -385,18 +401,18 @@ def parse_condition(self, query, prev_key=None, last_prev_key=None): else grouped_conditions | parse_condition ) yield grouped_conditions - elif key == '$in': + elif key == "$in": # use `any` to find with list, before comparing to single string grouped_conditions = Q(q, prev_key).any(value) for val in value: - for parse_condition in self.parse_condition({prev_key : val}): + for parse_condition in self.parse_condition({prev_key: val}): grouped_conditions = ( parse_condition if not grouped_conditions else grouped_conditions | parse_condition ) yield grouped_conditions - elif key == '$all': + elif key == "$all": yield Q(q, prev_key).all(value) else: yield Q(q, prev_key).any([value]) @@ -406,10 +422,7 @@ def parse_condition(self, query, prev_key=None, last_prev_key=None): def update(self, query, doc, *args, **kwargs): """BAckwards compatibility with update""" if isinstance(doc, list): - return [ - self.update_one(query, item, *args, **kwargs) - for item in doc - ] + return [self.update_one(query, item, *args, **kwargs) for item in doc] else: return self.update_one(query, doc, *args, **kwargs) @@ -438,8 +451,7 @@ def update_one(self, query, doc): return UpdateResult(raw_result=result) - def find(self, filter=None, sort=None, skip=None, limit=None, - *args, **kwargs): + def find(self, filter=None, sort=None, skip=None, limit=None, *args, **kwargs): """ Finds all matching results @@ -459,12 +471,7 @@ def find(self, filter=None, sort=None, skip=None, limit=None, except (AttributeError, TypeError): result = [] - result = TinyMongoCursor( - result, - sort=sort, - skip=skip, - limit=limit - ) + result = TinyMongoCursor(result, sort=sort, skip=skip, limit=limit) return result @@ -497,7 +504,7 @@ def delete_one(self, query): :return: DeleteResult """ item = self.find_one(query) - result = self.table.remove(where(u'_id') == item[u'_id']) + result = self.table.remove(where(u"_id") == item[u"_id"]) return DeleteResult(raw_result=result) @@ -509,10 +516,7 @@ def delete_many(self, query): :return: DeleteResult """ items = self.find(query) - result = [ - self.table.remove(where(u'_id') == item[u'_id']) - for item in items - ] + result = [self.table.remove(where(u"_id") == item[u"_id"]) for item in items] if query == {}: # need to reset TinyDB's index for docs order consistency @@ -561,7 +565,7 @@ def paginate(self, skip, limit): # {0: 20, 20: 40, 40: 60, 60: 62} if limit and limit < self.count(): limit = limits.get(skip, self.count()) - self.cursordat = self.cursordat[skip: limit] + self.cursordat = self.cursordat[skip:limit] def _order(self, value, is_reverse=None): """Parsing data to a sortable form @@ -589,12 +593,9 @@ def _list_parser(list_doc): return result # (TODO) include more data type - if value is None or not isinstance(value, (dict, - list, - basestring, - bool, - float, - int)): + if value is None or not isinstance( + value, (dict, list, basestring, bool, float, int) + ): # not support/sortable value type value = (0, None) @@ -644,25 +645,26 @@ def sort(self, key_or_list, direction=None): sort_specifier = list() if isinstance(key_or_list, list): if direction is not None: - raise ValueError('direction can not be set separately ' - 'if sorting by multiple fields.') + raise ValueError( + "direction can not be set separately " + "if sorting by multiple fields." + ) for pair in key_or_list: if not (isinstance(pair, list) or isinstance(pair, tuple)): - raise TypeError('key pair should be a list or tuple.') + raise TypeError("key pair should be a list or tuple.") if not len(pair) == 2: - raise ValueError('Need to be (key, direction) pair') + raise ValueError("Need to be (key, direction) pair") if not isinstance(pair[0], basestring): - raise TypeError('first item in each key pair must ' - 'be a string') + raise TypeError("first item in each key pair must " "be a string") if not isinstance(pair[1], int) or not abs(pair[1]) == 1: - raise TypeError('bad sort specification.') + raise TypeError("bad sort specification.") sort_specifier = key_or_list elif isinstance(key_or_list, basestring): if direction is not None: if not isinstance(direction, int) or not abs(direction) == 1: - raise TypeError('bad sort specification.') + raise TypeError("bad sort specification.") else: # default ASCENDING direction = 1 @@ -670,8 +672,10 @@ def sort(self, key_or_list, direction=None): sort_specifier = [(key_or_list, direction)] else: - raise ValueError('Wrong input, pass a field name and a direction,' - ' or pass a list of (key, direction) pairs.') + raise ValueError( + "Wrong input, pass a field name and a direction," + " or pass a list of (key, direction) pairs." + ) # sorting @@ -681,14 +685,14 @@ def sort(self, key_or_list, direction=None): pre_sect_stack = list() for pair in sort_specifier: - is_reverse = bool(1-pair[1]) + is_reverse = bool(1 - pair[1]) value_stack = list() for index, data in enumerate(_cursordat): # get field value not_found = None - for key in pair[0].split('.'): + for key in pair[0].split("."): not_found = True if isinstance(data, dict) and key in data: @@ -794,6 +798,7 @@ def count(self, with_limit_and_skip=False): class TinyGridFS(object): """GridFS for tinyDB""" + def __init__(self, *args, **kwargs): self.database = None From 78bbaa40b5d85135b551442668b5a70af1952811 Mon Sep 17 00:00:00 2001 From: FuzzLightyear <71232482+FuzzLightyear@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:41:07 -0400 Subject: [PATCH 2/4] Update tinymongo.py Merging (manually) various updates from other forks --- tinymongo/tinymongo.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tinymongo/tinymongo.py b/tinymongo/tinymongo.py index 1f69e01..d9a6269 100644 --- a/tinymongo/tinymongo.py +++ b/tinymongo/tinymongo.py @@ -152,6 +152,13 @@ def count(self): """ return self.find().count() + def count_documents(self, filter=None): + """ + Counts the documents in the collection. + :return: Integer representing the number of documents in the collection. + """ + return self.find(filter).count() + def drop(self, **kwargs): """ Removes a collection from the database. @@ -426,7 +433,7 @@ def update(self, query, doc, *args, **kwargs): else: return self.update_one(query, doc, *args, **kwargs) - def update_one(self, query, doc): + def update_one(self, query, doc, *args, **kwargs): """ Updates one element of the collection @@ -795,6 +802,20 @@ def count(self, with_limit_and_skip=False): """ return len(self.cursordat) + def __iter__(self): + self.cursorpos = -1 + return self + + def __next__(self): + """ + Returns the next record + :return: + """ + if not self.hasNext(): + raise StopIteration + self.cursorpos += 1 + return self.cursordat[self.cursorpos] + class TinyGridFS(object): """GridFS for tinyDB""" From 29ed85bac07b4ba56a33259e90050f47fc4ceac8 Mon Sep 17 00:00:00 2001 From: FuzzLightyear <71232482+FuzzLightyear@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:51:35 -0400 Subject: [PATCH 3/4] Update tinymongo.py ++ changes to tinymongo.py copied from JanSurft commit --- tinymongo/tinymongo.py | 77 ++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/tinymongo/tinymongo.py b/tinymongo/tinymongo.py index d9a6269..c8276fa 100644 --- a/tinymongo/tinymongo.py +++ b/tinymongo/tinymongo.py @@ -12,9 +12,10 @@ from uuid import uuid1 from tinydb import Query, TinyDB, where -from .results import InsertOneResult, InsertManyResult, UpdateResult, DeleteResult -from .errors import DuplicateKeyError - +# from .results import InsertOneResult, InsertManyResult, UpdateResult, DeleteResult +# from .errors import DuplicateKeyError +from tinymongodb.results import InsertOneResult, InsertManyResult, UpdateResult, DeleteResult +from tinymongodb.errors import DuplicateKeyError try: basestring except NameError: @@ -70,6 +71,9 @@ def _storage(self): def __getitem__(self, key): """Gets a new or existing database based in key""" + return self._get_db(key) + + def _get_db(self, key): return TinyMongoDatabase(key, self._foldername, self._storage) def close(self): @@ -78,7 +82,8 @@ def close(self): def __getattr__(self, name): """Gets a new or existing database based in attribute""" - return TinyMongoDatabase(name, self._foldername, self._storage) + # return TinyMongoDatabase(name, self._foldername, self._storage) + return self._get_db(name) class TinyMongoDatabase(object): @@ -458,20 +463,21 @@ def update_one(self, query, doc, *args, **kwargs): return UpdateResult(raw_result=result) - def find(self, filter=None, sort=None, skip=None, limit=None, *args, **kwargs): + def find(self, _filter=None, sort=None, skip=None, limit=None, *args, **kwargs): """ Finds all matching results - :param query: dictionary representing the mongo query + :param _filter: dictionary representing the mongo query + :type _filter: Optional[dict] :return: cursor containing the search results """ if self.table is None: self.build_table() - if filter is None: + if _filter is None: result = self.table.all() else: - allcond = self.parse_query(filter) + allcond = self.parse_query(_filter) try: result = self.table.search(allcond) @@ -482,7 +488,7 @@ def find(self, filter=None, sort=None, skip=None, limit=None, *args, **kwargs): return result - def find_one(self, filter=None): + def find_one(self, _filter=None): """ Finds one matching query element @@ -493,7 +499,7 @@ def find_one(self, filter=None): if self.table is None: self.build_table() - allcond = self.parse_query(filter) + allcond = self.parse_query(_filter) return self.table.get(allcond) @@ -601,7 +607,8 @@ def _list_parser(list_doc): # (TODO) include more data type if value is None or not isinstance( - value, (dict, list, basestring, bool, float, int) + # value, (dict, list, basestring, bool, float, int) + value, (dict, list, str, bool, float, int) ): # not support/sortable value type value = (0, None) @@ -612,7 +619,9 @@ def _list_parser(list_doc): elif isinstance(value, (int, float)): value = (1, value) - elif isinstance(value, basestring): + # elif isinstance(value, basestring): + elif isinstance(value, str): + value = (2, value) elif isinstance(value, dict): @@ -641,9 +650,11 @@ def sort(self, key_or_list, direction=None): Sorts a cursor object based on the input :param key_or_list: a list/tuple containing the sort specification, - i.e. ('user_number': -1), or a basestring + # i.e. ('user_number': -1), or a basestring + i.e. ('user_number': -1), or a str :param direction: sorting direction, 1 or -1, needed if key_or_list - is a basestring + # is a basestring + is a str :return: """ @@ -661,14 +672,16 @@ def sort(self, key_or_list, direction=None): raise TypeError("key pair should be a list or tuple.") if not len(pair) == 2: raise ValueError("Need to be (key, direction) pair") - if not isinstance(pair[0], basestring): + # if not isinstance(pair[0], basestring): + if not isinstance(pair[0], str): raise TypeError("first item in each key pair must " "be a string") if not isinstance(pair[1], int) or not abs(pair[1]) == 1: raise TypeError("bad sort specification.") sort_specifier = key_or_list - elif isinstance(key_or_list, basestring): + # elif isinstance(key_or_list, basestring): + elif isinstance(key_or_list, str): if direction is not None: if not isinstance(direction, int) or not abs(direction) == 1: raise TypeError("bad sort specification.") @@ -772,6 +785,19 @@ def sort(self, key_or_list, direction=None): return self + def limit(self, n): + self.cursordat = self.cursordat[:n] + return self + + def has_next(self): + """ + Returns True if the cursor has a next position, False if not + :return: + """ + cursor_pos = self.cursorpos + 1 + + return cursor_pos + 1 < len(self.cursordat) + def hasNext(self): """ Returns True if the cursor has a next position, False if not @@ -823,16 +849,25 @@ class TinyGridFS(object): def __init__(self, *args, **kwargs): self.database = None - def GridFS(self, tinydatabase): + def grid_fs(self, tinydatabase): """TODO: Must implement yet""" self.database = tinydatabase return self + def GridFS(self, tinydatabase): + """TODO: Must implement yet""" + self.database = tinydatabase + return self def generate_id(): """Generate new UUID""" # TODO: Use six.string_type to Py3 compat - try: - return unicode(uuid1()).replace(u"-", u"") - except NameError: - return str(uuid1()).replace(u"-", u"") + return str(uuid1()).replace(u"-", u"") + +# def generate_id(): +# """Generate new UUID""" +# # TODO: Use six.string_type to Py3 compat +# try: +# return unicode(uuid1()).replace(u"-", u"") +# except NameError: +# return str(uuid1()).replace(u"-", u"") From 88a09190ff71b68ed3928d0fab7e6c6a97242090 Mon Sep 17 00:00:00 2001 From: FuzzLightyear <71232482+FuzzLightyear@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:55:36 -0400 Subject: [PATCH 4/4] Update tinymongo.py maybe want JSONStorage. should test. Copied from @sumants-dev https://github.com/sumants-dev --- tinymongo/tinymongo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tinymongo/tinymongo.py b/tinymongo/tinymongo.py index c8276fa..448f0e0 100644 --- a/tinymongo/tinymongo.py +++ b/tinymongo/tinymongo.py @@ -67,6 +67,7 @@ def _storage(self): return serialization """ + # return storages.JSONStorage return TinyDB.default_storage_class def __getitem__(self, key):