From b52a27d8d15350dc29cf5630eecf48c2f88e0edf Mon Sep 17 00:00:00 2001 From: Jwink3101 Date: Fri, 26 Oct 2018 18:02:39 -0600 Subject: [PATCH 1/2] Still not working yet. Not sure what is causing the deadlock... --- sqlitedict.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/sqlitedict.py b/sqlitedict.py index 7b60235..496f339 100755 --- a/sqlitedict.py +++ b/sqlitedict.py @@ -63,6 +63,7 @@ def exec_(_code_, _globs_=None, _locs_=None): exec_("def reraise(tp, value, tb=None):\n" " raise tp, value, tb\n") else: + unicode = str def reraise(tp, value, tb=None): if value is None: value = tp() @@ -104,12 +105,13 @@ def decode(obj): """Deserialize objects retrieved from SQLite.""" return loads(bytes(obj)) - class SqliteDict(DictClass): VALID_FLAGS = ['c', 'r', 'w', 'n'] def __init__(self, filename=None, tablename='unnamed', flag='c', - autocommit=False, journal_mode="DELETE", encode=encode, decode=decode): + autocommit=False, journal_mode="DELETE", + encode=encode, decode=decode, + _parent=None,_subtable=None): """ Initialize a thread-safe sqlite-backed dictionary. The dictionary will be a table `tablename` in database file `filename`. A single file (=database) @@ -141,6 +143,24 @@ def __init__(self, filename=None, tablename='unnamed', flag='c', The default is to use pickle. """ + # Handle if it has a parent and skip the rest + self._parent = _parent + self._subtable = _subtable + if self._parent is not None: + if self._subtable: + self.tablename = self._subtable + else: + self.tablename = _parent.tablename + '_' + hex(random.randint(0, 0xffffff))[2:] + + for attr in ['filename','autocommit','journal_mode', + '_encode','_decode','conn','flag']: + setattr(self,attr,getattr(_parent,attr)) + logger.info("opening Sqlite table %r in %s" % (tablename, filename)) + MAKE_TABLE = 'CREATE TABLE IF NOT EXISTS "%s" (key TEXT PRIMARY KEY, value BLOB)' % self.tablename + self._parent.commit() + self.conn.execute(MAKE_TABLE) + return + self.in_temp = filename is None if self.in_temp: randpart = hex(random.randint(0, 0xffffff))[2:] @@ -165,8 +185,8 @@ def __init__(self, filename=None, tablename='unnamed', flag='c', self.tablename = tablename self.autocommit = autocommit self.journal_mode = journal_mode - self.encode = encode - self.decode = decode + self._encode = encode + self._decode = decode logger.info("opening Sqlite table %r in %s" % (tablename, filename)) MAKE_TABLE = 'CREATE TABLE IF NOT EXISTS "%s" (key TEXT PRIMARY KEY, value BLOB)' % self.tablename @@ -250,14 +270,21 @@ def __setitem__(self, key, value): raise RuntimeError('Refusing to write to read-only SqliteDict') ADD_ITEM = 'REPLACE INTO "%s" (key, value) VALUES (?,?)' % self.tablename - self.conn.execute(ADD_ITEM, (key, self.encode(value))) - + val = self.encode(value) + self.conn.execute(ADD_ITEM, (key, val)) + print(val) def __delitem__(self, key): if self.flag == 'r': raise RuntimeError('Refusing to delete from read-only SqliteDict') if key not in self: raise KeyError(key) + + # Handle if subdict + obj = self[key] + if isinstance(obj,SqliteDict) and obj._parent is not None: + obj.clear() + DEL_ITEM = 'DELETE FROM "%s" WHERE key = ?' % self.tablename self.conn.execute(DEL_ITEM, (key,)) @@ -355,6 +382,29 @@ def __del__(self): # closed after connection lost (exceptions are always ignored # in __del__ method. pass + + def encode(self,obj): + if isinstance(obj,SqliteDict) and obj._parent is not None: + # Just store a reference to the tablename + obj = '__nested:' + obj.tablename + print('a') + return self._encode(obj) + + def decode(self,obj): + obj = self._decode(obj) + if not (isinstance(obj,(str,unicode)) and obj.startswith('__nested:')): + return obj + _subtable = obj[len('__nested:'):] + return self.subdict(_subtable=_subtable) + + def subdict(self,_subtable=None): + """ + Return a dictionary that is *also* a SqliteDict + """ + # Note that all other attributed are automatically taken from + # the parent so no need to enumerate them here + return SqliteDict(_parent=self,_subtable=_subtable) + # Adding extra methods for python 2 compatibility (at import time) if major_version == 2: From 1bcf9caee6d1b6c9f4d1a95862d8e18bd4f2f780 Mon Sep 17 00:00:00 2001 From: Jwink3101 Date: Mon, 29 Oct 2018 13:55:01 -0600 Subject: [PATCH 2/2] I *think* I fixed it all --- sqlitedict.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sqlitedict.py b/sqlitedict.py index 496f339..e693b2b 100755 --- a/sqlitedict.py +++ b/sqlitedict.py @@ -272,7 +272,7 @@ def __setitem__(self, key, value): ADD_ITEM = 'REPLACE INTO "%s" (key, value) VALUES (?,?)' % self.tablename val = self.encode(value) self.conn.execute(ADD_ITEM, (key, val)) - print(val) + def __delitem__(self, key): if self.flag == 'r': raise RuntimeError('Refusing to delete from read-only SqliteDict') @@ -283,8 +283,9 @@ def __delitem__(self, key): # Handle if subdict obj = self[key] if isinstance(obj,SqliteDict) and obj._parent is not None: - obj.clear() - + DEL_TAB = 'DROP TABLE "%s"' % obj.tablename + self.conn.execute(DEL_TAB, tuple()) + DEL_ITEM = 'DELETE FROM "%s" WHERE key = ?' % self.tablename self.conn.execute(DEL_ITEM, (key,)) @@ -374,6 +375,10 @@ def terminate(self): logger.exception("failed to delete %s" % (self.filename)) def __del__(self): + # Not sure why but when creating a subdict, it tries to close itself + # so ignore subdicts + if self._parent is not None: + return # like close(), but assume globals are gone by now (do not log!) try: self.close(do_log=False, force=True) @@ -387,7 +392,6 @@ def encode(self,obj): if isinstance(obj,SqliteDict) and obj._parent is not None: # Just store a reference to the tablename obj = '__nested:' + obj.tablename - print('a') return self._encode(obj) def decode(self,obj): @@ -412,7 +416,6 @@ def subdict(self,_subtable=None): del SqliteDict.__bool__ # not needed and confusing #endclass SqliteDict - class SqliteMultithread(Thread): """ Wrap sqlite connection in a way that allows concurrent requests from multiple threads.