From 594fe2504b85fbefc37ab443c6e07fdd28d79f8f Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 11 Oct 2019 19:06:31 +0300 Subject: [PATCH 1/9] #38 URI based construction implemented. --- py2store/persisters/_arangodb_in_progress.py | 23 ++++----- py2store/persisters/_couchdb_in_progress.py | 0 py2store/persisters/couchdb_w_couchdb.py | 23 +++------ py2store/persisters/dropbox_w_dropbox.py | 12 +++-- py2store/persisters/dropbox_w_requests.py | 12 ++--- py2store/persisters/ftp_persister.py | 29 ++++++----- py2store/persisters/mongo_w_pymongo.py | 23 ++++----- py2store/persisters/sql_w_odbc.py | 52 +++++++++++--------- py2store/persisters/sql_w_sqlalchemy.py | 16 +++--- py2store/persisters/ssh_persister.py | 35 ++++++++++--- tests/test_dropbox_w_requests.py | 4 +- tests/test_sql_w_sqlalchemy.py | 8 +-- 12 files changed, 132 insertions(+), 105 deletions(-) delete mode 100644 py2store/persisters/_couchdb_in_progress.py diff --git a/py2store/persisters/_arangodb_in_progress.py b/py2store/persisters/_arangodb_in_progress.py index 57470f0..2be223e 100644 --- a/py2store/persisters/_arangodb_in_progress.py +++ b/py2store/persisters/_arangodb_in_progress.py @@ -57,22 +57,19 @@ class ArangoDbPersister(Persister): def __init__( self, - user='root', - password='root', - url='http://127.0.0.1:8529', - db_name='py2store', - collection_name='test', + uri, # Example dict( + # user='root', + # password='root', + # url='http://127.0.0.1:8529', + # db_name='py2store', + # ) + collection='test', key_fields=('key',), # _id, _key and _rev are reserved by db key_fields_separator='::', ): - self._connection = Connection( - arangoURL=url, - username=user, - password=password, - ) - - self._db_name = db_name - self._collection_name = collection_name + self._db_name = uri.pop('db_name') + self._connection = Connection(**uri) + self._collection_name = collection # If DB not created: if not self._connection.hasDatabase(self._db_name): diff --git a/py2store/persisters/_couchdb_in_progress.py b/py2store/persisters/_couchdb_in_progress.py deleted file mode 100644 index e69de29..0000000 diff --git a/py2store/persisters/couchdb_w_couchdb.py b/py2store/persisters/couchdb_w_couchdb.py index de1be70..364cf6d 100644 --- a/py2store/persisters/couchdb_w_couchdb.py +++ b/py2store/persisters/couchdb_w_couchdb.py @@ -65,27 +65,20 @@ def clear(self): def __init__( self, - user='admin', - password='admin', - url='http://127.0.0.1:5984', - db_name='py2store', + uri, # Example: "https://username:password@example.com:5984/" + collection='py2store', key_fields=('_id',), couchdb_client_kwargs=None ): if couchdb_client_kwargs is None: couchdb_client_kwargs = {} - if user and password: - # put credentials in url if provided like https://username:password@example.com:5984/ - if '//' in url: # if scheme present - url = f'{url.split("//")[0]}//{user}:{password}@{url.split("//")[1]}' - else: - url = f'http//{user}:{password}@{url}' - self._couchdb_server = Server(url=url, **couchdb_client_kwargs) - self._db_name = db_name + + self._couchdb_server = Server(url=uri, **couchdb_client_kwargs) + self._db_name = collection # if db not created - if db_name not in self._couchdb_server: - self._couchdb_server.create(db_name) - self._cdb = self._couchdb_server[db_name] + if collection not in self._couchdb_server: + self._couchdb_server.create(collection) + self._cdb = self._couchdb_server[collection] if isinstance(key_fields, str): key_fields = (key_fields,) diff --git a/py2store/persisters/dropbox_w_dropbox.py b/py2store/persisters/dropbox_w_dropbox.py index faddfcb..d11e8bc 100644 --- a/py2store/persisters/dropbox_w_dropbox.py +++ b/py2store/persisters/dropbox_w_dropbox.py @@ -49,9 +49,15 @@ class DropboxPersister(Persister): >>> del s['/py2store_data/test/_can_remove'] """ - def __init__(self, rootdir, oauth2_access_token, - connection_kwargs=None, files_upload_kwargs=None, - files_list_folder_kwargs=None, rev=None): + def __init__( + self, + rootdir, + oauth2_access_token, + connection_kwargs=None, + files_upload_kwargs=None, + files_list_folder_kwargs=None, + rev=None, + ): if connection_kwargs is None: connection_kwargs = {} diff --git a/py2store/persisters/dropbox_w_requests.py b/py2store/persisters/dropbox_w_requests.py index c273420..d25ef17 100644 --- a/py2store/persisters/dropbox_w_requests.py +++ b/py2store/persisters/dropbox_w_requests.py @@ -12,9 +12,9 @@ class DropboxFolderCopyReader(Reader): """Makes a full local copy of the folder (by default, to a local temp folder) and gives access to it. """ - def __init__(self, url, path=tempfile.gettempdir()): - self.url = url - self.path = path + def __init__(self, uri, collection=tempfile.gettempdir()): + self.url = uri + self.path = collection os.makedirs(self.path, exist_ok=True) self._zip_filepath = os.path.join(self.path, 'shared_folder.zip') @@ -51,9 +51,9 @@ def _unzip(self): class DropboxFileCopyReader(Reader): - def __init__(self, url, path=None): - self.url = url - self.path = path or self._get_filename_from_url() + def __init__(self, uri, collection=None): + self.url = uri + self.path = collection or self._get_filename_from_url() download_from_dropbox(self.url, self.path, as_zip=False) self.file = open(self.path, 'r') diff --git a/py2store/persisters/ftp_persister.py b/py2store/persisters/ftp_persister.py index 53e2dc8..0f78e33 100644 --- a/py2store/persisters/ftp_persister.py +++ b/py2store/persisters/ftp_persister.py @@ -1,7 +1,8 @@ from py2store.base import Persister from ftplib import FTP, all_errors import os.path -from io import BytesIO +from io import BytesIO + def remote_mkdir(ftp, remote_directory): """ @@ -16,7 +17,7 @@ def remote_mkdir(ftp, remote_directory): # top-level relative directory must exist return False try: - ftp.cwd(remote_directory) # sub-directory exists + ftp.cwd(remote_directory) # sub-directory exists except all_errors: dirname, basename = os.path.split(remote_directory.rstrip('/')) remote_mkdir(ftp, dirname) # make parent directories @@ -58,15 +59,19 @@ class FtpPersister(Persister): 0 """ - def __init__(self, - user='dlpuser@dlptest.com', - password='fLDScD4Ynth0p4OJ6bW6qCxjh', - url='ftp.dlptest.com', - rootdir='./py2store', - encoding='utf8' - ): - self._ftp = FTP(host=url, user=user, passwd=password) - self._rootdir = rootdir + def __init__( + self, + uri, + # Example dict( + # user='dlpuser@dlptest.com', + # password='fLDScD4Ynth0p4OJ6bW6qCxjh', + # url='ftp.dlptest.com', + # ) + collection='./py2store', + encoding='utf8' + ): + self._ftp = FTP(**uri) + self._rootdir = collection self._encoding = encoding self._ftp.encoding = encoding remote_mkdir(self._ftp, self._rootdir) @@ -114,4 +119,4 @@ def __del__(self): """ Close ssh session when an object is deleted """ - self._ftp.close() \ No newline at end of file + self._ftp.close() diff --git a/py2store/persisters/mongo_w_pymongo.py b/py2store/persisters/mongo_w_pymongo.py index a745022..0d7467f 100644 --- a/py2store/persisters/mongo_w_pymongo.py +++ b/py2store/persisters/mongo_w_pymongo.py @@ -55,18 +55,18 @@ class MongoPersister(Persister): {'first': 'Vitalik', 'last': 'Buterin'} --> {'yob': 1994, 'proj': 'ethereum', 'bdfl': True} """ - def clear(self): - raise NotImplementedError("clear is disabled by default, for your own protection! " - "Loop and delete if you really want to.") - - def __init__(self, db_name='py2store', collection_name='test', key_fields=('_id',), data_fields=None, - mongo_client_kwargs=None): - if mongo_client_kwargs is None: - mongo_client_kwargs = {} - self._mongo_client = MongoClient(**mongo_client_kwargs) + def __init__( + self, + uri, + collection='test', + key_fields=('_id',), + data_fields=None, + ): + db_name = uri.pop('db_name') + self._mongo_client = MongoClient(**uri) self._db_name = db_name - self._collection_name = collection_name - self._mgc = self._mongo_client[db_name][collection_name] + self._collection_name = collection + self._mgc = self._mongo_client[db_name][collection] if isinstance(key_fields, str): key_fields = (key_fields,) if data_fields is None: @@ -81,6 +81,7 @@ def __init__(self, db_name='py2store', collection_name='test', key_fields=('_id' data_fields = {k: True for k in data_fields} if '_id' not in data_fields: data_fields['_id'] = False + self._data_fields = data_fields self._key_fields = key_fields diff --git a/py2store/persisters/sql_w_odbc.py b/py2store/persisters/sql_w_odbc.py index 3c7fc9f..c1b20be 100644 --- a/py2store/persisters/sql_w_odbc.py +++ b/py2store/persisters/sql_w_odbc.py @@ -2,21 +2,40 @@ import subprocess from collections.abc import MutableMapping +from py2store.util import ModuleNotFoundErrorNiceMessage + +with ModuleNotFoundErrorNiceMessage(): + import pyodbc -class SQLServerPersister(MutableMapping): - def __init__(self, conn_protocol='tcp', host='localhost', port='1433', db_username='SA', db_pass='Admin123x', - db_name='py2store', table_name='person', primary_key='id', data_fields=('name',)): +class SQLServerPersister(MutableMapping): + def __init__( + self, + uri, + # Example: dict( + # conn_protocol='tcp', + # host='localhost', + # port='1433', + # db_username='SA', + # db_pass='Admin123x', + # db_name='py2store', + # ) + collection='py2store_default_table', + primary_key='id', + data_fields=('name',) + ): self.__check_dependencies() - self._sql_server_client = pyodbc.connect('DRIVER={{ODBC Driver 17 for SQL Server}};' - 'SERVER={}:{},{};' - 'DATABASE={};' - 'UID={};' - 'PWD={}' - .format(conn_protocol, host, port, db_name, db_username, db_pass)) + self._sql_server_client = pyodbc.connect( + 'DRIVER={{ODBC Driver 17 for SQL Server}};' + 'SERVER={conn_protocol}:{host},{port};' + 'DATABASE={db_name};' + 'UID={db_username};' + 'PWD={db_pass}' + .format(**uri) + ) self._cursor = self._sql_server_client.cursor() - self._table_name = table_name + self._table_name = collection self._primary_key = primary_key self._select_all_query = "SELECT * from {table};".format(table=self._table_name) @@ -28,16 +47,6 @@ def __init__(self, conn_protocol='tcp', host='localhost', port='1433', db_userna @staticmethod def __check_dependencies(): - import pkg_resources - installed_packages = [pkg.project_name for pkg in pkg_resources.working_set] - if 'pyodbc' not in installed_packages: - raise ModuleNotFoundError("'SQLServerPersister' depends on the module 'pyodbc' which is not installed. " - "Try installing dependency using 'pip install pyodbc'.") - - # import pyodbc globally - global pyodbc - import pyodbc - if 'ubuntu' in platform.platform().lower(): result = subprocess.Popen(["dpkg", "-s", "msodbcsql17"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = result.communicate() @@ -115,6 +124,3 @@ def test_sqlserver_persister(): print("Getting the length") print(len(sql_server_persister)) - - -test_sqlserver_persister() diff --git a/py2store/persisters/sql_w_sqlalchemy.py b/py2store/persisters/sql_w_sqlalchemy.py index 42aef71..21f40f7 100644 --- a/py2store/persisters/sql_w_sqlalchemy.py +++ b/py2store/persisters/sql_w_sqlalchemy.py @@ -15,15 +15,15 @@ class SQLAlchemyPersister(Persister): def __init__( self, - db_uri='sqlite:///my_sqlite.db', - collection_name='py2store_default_table', + uri='sqlite:///my_sqlite.db', + collection='py2store_default_table', key_fields=('id',), data_fields=('data',), autocommit=True, **db_kwargs ): """ - :param db_uri: Uniform Resource Identifier of a database you would like to use. + :param uri: Uniform Resource Identifier of a database you would like to use. Unix/Mac (note the four leading slashes) sqlite:////absolute/path/to/foo.db @@ -41,7 +41,7 @@ def __init__( I.e. in general: dialect+driver://username:password@host:port/database - :param collection_name: name of the table to use, i.e. "my_table". + :param collection: name of the table to use, i.e. "my_table". :param key_fields: indexed keys columns names. :param data_fields: non-indexed data columns names. :param autocommit: whether each data change should be instantly commited, or not. @@ -57,15 +57,15 @@ def __init__( self.table = None self.session = None - self.setup(db_uri, collection_name, **db_kwargs) + self.setup(uri, collection, **db_kwargs) - def setup(self, db_uri, collection_name, **db_kwargs): + def setup(self, uri, collection, **db_kwargs): # Setup connection to our DB: - engine = create_engine(db_uri, **db_kwargs) + engine = create_engine(uri, **db_kwargs) self.connection = engine.connect() # Create a table: - self.table = self._create_table(collection_name, engine) + self.table = self._create_table(collection, engine) # Open ORM session: self.session = sessionmaker(bind=engine)() diff --git a/py2store/persisters/ssh_persister.py b/py2store/persisters/ssh_persister.py index 56170e2..7578b88 100644 --- a/py2store/persisters/ssh_persister.py +++ b/py2store/persisters/ssh_persister.py @@ -66,17 +66,36 @@ class SshPersister(Persister): def __init__( self, - user='stud', - password='stud', - url='10.1.103.201', - rootdir='./py2store', - encoding='utf8' - ): + uri, # Example: dict(url='10.1.103.201', user='stud', password='stud') + collection='./py2store', + encoding='utf8', + ): + """ + :param uri: A dict of the following key-values: + :param str hostname: the server to connect to, REQUIRED, others are optional + :param int port: the server port to connect to + :param str username: + the username to authenticate as (defaults to the current local + username) + :param str password: + Used for password authentication; is also used for private key + decryption if ``passphrase`` is not given. + :param str passphrase: + Used for decrypting private keys. + :param .PKey pkey: an optional private key to use for authentication + :param str key_filename: + the filename, or list of filenames, of optional private key(s) + and/or certs to try for authentication + :param float timeout: + an optional timeout (in + :param collection: root direction + :param encoding: + """ self._ssh = paramiko.SSHClient() self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self._ssh.connect(url, username=user, password=password) + self._ssh.connect(**uri) self._sftp = self._ssh.open_sftp() - self._rootdir = rootdir + self._rootdir = collection self._encoding = encoding remote_mkdir(self._sftp, self._rootdir) diff --git a/tests/test_dropbox_w_requests.py b/tests/test_dropbox_w_requests.py index 13fc5e3..315af69 100644 --- a/tests/test_dropbox_w_requests.py +++ b/tests/test_dropbox_w_requests.py @@ -13,8 +13,8 @@ def shared_folder_persister(): path = 'tests/data/path' persister = DropboxFolderCopyReader( - url=SHARED_FOLDER_URL, - path=path, + uri=SHARED_FOLDER_URL, + collection=path, ) yield persister try: diff --git a/tests/test_sql_w_sqlalchemy.py b/tests/test_sql_w_sqlalchemy.py index 26db18d..02d87a7 100644 --- a/tests/test_sql_w_sqlalchemy.py +++ b/tests/test_sql_w_sqlalchemy.py @@ -1,6 +1,6 @@ from py2store.persisters.sql_w_sqlalchemy import SQLAlchemyPersister -from py2store.stores.sql_w_sqlalchemy import SQLAlchemyTupleStore, SQLAlchemyStore -from tests.base_test import BaseStoreTest, BasePersisterTest, BaseKeyTupleStoreTest, BaseTupleStoreTest +from py2store.stores.sql_w_sqlalchemy import SQLAlchemyTupleStore +from tests.base_test import BasePersisterTest, BaseTupleStoreTest SQLITE_DB_URI = 'sqlite:///:memory:' SQLITE_TABLE_NAME = 'test_table' @@ -8,8 +8,8 @@ class TestSQLAlchemyPersister(BasePersisterTest): db = SQLAlchemyPersister( - db_uri=SQLITE_DB_URI, - collection_name=SQLITE_TABLE_NAME, + uri=SQLITE_DB_URI, + collection=SQLITE_TABLE_NAME, key_fields=list(BasePersisterTest.key.keys()), data_fields=list(BasePersisterTest.data.keys()), ) From 28794334c62a7b97c0d96afc223d4e8dc93437db Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 11 Oct 2019 19:14:28 +0300 Subject: [PATCH 2/9] Merge. --- py2store/persisters/mongo_w_pymongo.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/py2store/persisters/mongo_w_pymongo.py b/py2store/persisters/mongo_w_pymongo.py index 3ea2481..704c576 100644 --- a/py2store/persisters/mongo_w_pymongo.py +++ b/py2store/persisters/mongo_w_pymongo.py @@ -55,12 +55,17 @@ class MongoPersister(Persister): {'first': 'Vitalik', 'last': 'Buterin'} --> {'yob': 1994, 'proj': 'ethereum', 'bdfl': True} """ - def __init__(self, db_name='py2store', collection_name='test', key_fields=('_id',), data_fields=None, - mongo_client_kwargs=None): - if mongo_client_kwargs is None: - mongo_client_kwargs = {} - self._mongo_client = MongoClient(**mongo_client_kwargs) + + def __init__( + self, + uri, + collection='test', + key_fields=('_id',), + data_fields=None, + ): + db_name = uri.pop('db_name') self._db_name = db_name + self._mongo_client = MongoClient(**uri) self._collection_name = collection self._mgc = self._mongo_client[db_name][collection] if isinstance(key_fields, str): From f3d49c740c0838a3090697ce1799b76568d228ba Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Thu, 31 Oct 2019 16:28:20 +0300 Subject: [PATCH 3/9] Init with string URI. Classmethod .from_kwargs to init with params. --- py2store/persisters/couchdb_w_couchdb.py | 6 +++ py2store/persisters/dropbox_w_dropbox.py | 12 +++--- py2store/persisters/dropbox_w_requests.py | 8 ++++ py2store/persisters/ftp_persister.py | 20 +++++++--- py2store/persisters/mongo_w_pymongo.py | 17 ++++++--- py2store/persisters/sql_w_odbc.py | 46 +++++++++++++---------- py2store/persisters/sql_w_sqlalchemy.py | 24 +++++++++--- py2store/persisters/ssh_persister.py | 37 +++++++++--------- py2store/utils/uri_parsing.py | 42 +++++++++++++++++++++ tests/test_sql_w_sqlalchemy.py | 14 ++++++- 10 files changed, 163 insertions(+), 63 deletions(-) create mode 100644 py2store/utils/uri_parsing.py diff --git a/py2store/persisters/couchdb_w_couchdb.py b/py2store/persisters/couchdb_w_couchdb.py index 364cf6d..9b20b5e 100644 --- a/py2store/persisters/couchdb_w_couchdb.py +++ b/py2store/persisters/couchdb_w_couchdb.py @@ -1,5 +1,6 @@ from py2store.base import Persister from py2store.util import ModuleNotFoundErrorNiceMessage +from py2store.utils.uri_parsing import build_uri with ModuleNotFoundErrorNiceMessage(): from couchdb import Server @@ -84,6 +85,11 @@ def __init__( self._key_fields = key_fields + @classmethod + def from_kwargs(cls, username, password, host='localhost', port=5984, scheme='http', database='', **kwargs): + uri = build_uri(scheme, database, username, password, host, port) + return cls(uri, **kwargs) + def __get_item_internal(self, k): # some methods need original couch docs, so __getitem__ is divided on two methods k = self.__replace_internals(k) diff --git a/py2store/persisters/dropbox_w_dropbox.py b/py2store/persisters/dropbox_w_dropbox.py index d11e8bc..246ef0d 100644 --- a/py2store/persisters/dropbox_w_dropbox.py +++ b/py2store/persisters/dropbox_w_dropbox.py @@ -51,14 +51,13 @@ class DropboxPersister(Persister): def __init__( self, - rootdir, - oauth2_access_token, + uri='oauth2_access_token', + collection='my/root/dir', connection_kwargs=None, files_upload_kwargs=None, files_list_folder_kwargs=None, rev=None, ): - if connection_kwargs is None: connection_kwargs = {} if files_upload_kwargs is None: @@ -66,8 +65,11 @@ def __init__( if files_list_folder_kwargs is None: files_list_folder_kwargs = {'recursive': True, 'include_non_downloadable_files': False} - self._prefix = rootdir - self._con = Dropbox(oauth2_access_token, **connection_kwargs) + self._con = Dropbox( + uri, # OAuth token here + **connection_kwargs + ) + self._prefix = collection # aka root dir self._connection_kwargs = connection_kwargs self._files_upload_kwargs = files_upload_kwargs self._files_list_folder_kwargs = files_list_folder_kwargs diff --git a/py2store/persisters/dropbox_w_requests.py b/py2store/persisters/dropbox_w_requests.py index d25ef17..581c5a3 100644 --- a/py2store/persisters/dropbox_w_requests.py +++ b/py2store/persisters/dropbox_w_requests.py @@ -22,6 +22,10 @@ def __init__(self, uri, collection=tempfile.gettempdir()): self._files = [] self._get_folder() + @classmethod + def from_kwargs(cls, **kwargs): + return cls(**kwargs) + def __getitem__(self, rel_path): real_path = os.path.join(self.path, rel_path) try: @@ -58,6 +62,10 @@ def __init__(self, uri, collection=None): download_from_dropbox(self.url, self.path, as_zip=False) self.file = open(self.path, 'r') + @classmethod + def from_kwargs(cls, **kwargs): + return cls(**kwargs) + def __getitem__(self, index): self.file.seek(0) return self.readlines()[index] diff --git a/py2store/persisters/ftp_persister.py b/py2store/persisters/ftp_persister.py index 0f78e33..cf7562b 100644 --- a/py2store/persisters/ftp_persister.py +++ b/py2store/persisters/ftp_persister.py @@ -3,6 +3,8 @@ import os.path from io import BytesIO +from py2store.utils.uri_parsing import build_uri, parse_uri + def remote_mkdir(ftp, remote_directory): """ @@ -62,20 +64,26 @@ class FtpPersister(Persister): def __init__( self, uri, - # Example dict( - # user='dlpuser@dlptest.com', - # password='fLDScD4Ynth0p4OJ6bW6qCxjh', - # url='ftp.dlptest.com', - # ) collection='./py2store', encoding='utf8' ): - self._ftp = FTP(**uri) + uri_parsed = parse_uri(uri) + + self._ftp = FTP( + host=uri_parsed['host'], + user=uri_parsed['username'], + passwd=uri_parsed['password'], + ) self._rootdir = collection self._encoding = encoding self._ftp.encoding = encoding remote_mkdir(self._ftp, self._rootdir) + @classmethod + def from_kwargs(cls, host, port=21, username='', password='', scheme='tcp', **kwargs): + uri = build_uri(scheme, username=username, password=password, host=host, port=port) + return cls(uri, **kwargs) + def __getitem__(self, k): bio = BytesIO() self._ftp.retrbinary("RETR {}".format(k), bio.write) diff --git a/py2store/persisters/mongo_w_pymongo.py b/py2store/persisters/mongo_w_pymongo.py index 704c576..384489d 100644 --- a/py2store/persisters/mongo_w_pymongo.py +++ b/py2store/persisters/mongo_w_pymongo.py @@ -1,6 +1,7 @@ from py2store.base import Persister from py2store.util import ModuleNotFoundErrorNiceMessage +from py2store.utils.uri_parsing import build_uri, parse_uri with ModuleNotFoundErrorNiceMessage(): from pymongo import MongoClient @@ -55,7 +56,6 @@ class MongoPersister(Persister): {'first': 'Vitalik', 'last': 'Buterin'} --> {'yob': 1994, 'proj': 'ethereum', 'bdfl': True} """ - def __init__( self, uri, @@ -63,15 +63,17 @@ def __init__( key_fields=('_id',), data_fields=None, ): - db_name = uri.pop('db_name') + # db_name = uri.pop('db_name') + uri, db_name = uri.rsplit('/', 1) + uri_parsed = parse_uri(uri) + self._db_name = db_name - self._mongo_client = MongoClient(**uri) + self._mongo_client = MongoClient(**uri_parsed) self._collection_name = collection self._mgc = self._mongo_client[db_name][collection] + if isinstance(key_fields, str): key_fields = (key_fields,) - if data_fields is None: - pass self._key_projection = {k: True for k in key_fields} if '_id' not in key_fields: @@ -86,6 +88,11 @@ def __init__( self._data_fields = data_fields self._key_fields = key_fields + @classmethod + def from_kwargs(cls, database, username, password, host='localhost', port=1433, scheme='mongodb', **kwargs): + uri = build_uri(scheme, database, username, password, host, port) + return cls(uri, **kwargs) + def __getitem__(self, k): doc = self._mgc.find_one(k, projection=self._data_fields) if doc is not None: diff --git a/py2store/persisters/sql_w_odbc.py b/py2store/persisters/sql_w_odbc.py index c1b20be..648f0eb 100644 --- a/py2store/persisters/sql_w_odbc.py +++ b/py2store/persisters/sql_w_odbc.py @@ -3,35 +3,36 @@ from collections.abc import MutableMapping from py2store.util import ModuleNotFoundErrorNiceMessage +from py2store.utils.uri_parsing import parse_uri, build_uri with ModuleNotFoundErrorNiceMessage(): import pyodbc class SQLServerPersister(MutableMapping): - def __init__( - self, - uri, - # Example: dict( - # conn_protocol='tcp', - # host='localhost', - # port='1433', - # db_username='SA', - # db_pass='Admin123x', - # db_name='py2store', - # ) - collection='py2store_default_table', - primary_key='id', - data_fields=('name',) - ): + def __init__(self, uri, collection='py2store_default_table', primary_key='id', data_fields=('name',)): + """ + :param uri: Uniform Resource Identifier of a database you would like to use. + tcp://user:password@localhost:1433/my_db + or + user:password@localhost:1433/my_db + or even just + user:password/my_db + + :param collection: name of the table to use, i.e. "my_table". + :param primary_key: primary key column name. + :param data_fields: data columns names. + """ self.__check_dependencies() + + uri_parsed = parse_uri(uri) self._sql_server_client = pyodbc.connect( 'DRIVER={{ODBC Driver 17 for SQL Server}};' - 'SERVER={conn_protocol}:{host},{port};' - 'DATABASE={db_name};' - 'UID={db_username};' - 'PWD={db_pass}' - .format(**uri) + 'SERVER={scheme}:{host},{port};' + 'DATABASE={database};' + 'UID={username};' + 'PWD={password}' + .format(**uri_parsed) ) self._cursor = self._sql_server_client.cursor() @@ -45,6 +46,11 @@ def __init__( self._del_query = "DELETE from {table} where {primary_key} = {{value}};".format(table=self._table_name, primary_key=self._primary_key) + @classmethod + def from_kwargs(cls, database, username, password, host='localhost', port=1433, conn_protocol='tcp', **kwargs): + uri = build_uri(conn_protocol, database, username, password, host, port) + return cls(uri, **kwargs) + @staticmethod def __check_dependencies(): if 'ubuntu' in platform.platform().lower(): diff --git a/py2store/persisters/sql_w_sqlalchemy.py b/py2store/persisters/sql_w_sqlalchemy.py index 21f40f7..6594f54 100644 --- a/py2store/persisters/sql_w_sqlalchemy.py +++ b/py2store/persisters/sql_w_sqlalchemy.py @@ -1,4 +1,5 @@ from py2store.util import ModuleNotFoundErrorNiceMessage +from py2store.utils.uri_parsing import build_uri with ModuleNotFoundErrorNiceMessage(): from sqlalchemy import create_engine, Column, String, Table @@ -24,14 +25,14 @@ def __init__( ): """ :param uri: Uniform Resource Identifier of a database you would like to use. - Unix/Mac (note the four leading slashes) + Unix/Mac sqlite:////absolute/path/to/foo.db - Windows (note 3 leading forward slashes and backslash escapes) + Windows sqlite:///C:\\absolute\\path\\to\\foo.db Or go for in-memory DB with NO PERSISTANCE: - sqlite:///:memory: + sqlite:///:memory: Other options: postgresql://user:password@localhost:5432/my_db @@ -39,13 +40,13 @@ def __init__( oracle://user:password:tiger@localhost:1521/sidname I.e. in general: - dialect+driver://username:password@host:port/database + scheme://username:password@host:port/database :param collection: name of the table to use, i.e. "my_table". :param key_fields: indexed keys columns names. :param data_fields: non-indexed data columns names. - :param autocommit: whether each data change should be instantly commited, or not. - It's off for context manager usecase (tbd). + :param autocommit: whether each data change should be instantly + commited, or not. It's off for context manager usecase (tbd). :param kwargs: any extra kwargs for SQLAlchemy engine to setup. """ @@ -59,6 +60,17 @@ def __init__( self.setup(uri, collection, **db_kwargs) + @classmethod + def from_kwargs(cls, scheme='sqlite', database='my_sqlite.db', + username=None, password=None, + host='localhost', port=None, **kwargs): + if scheme == 'sqlite': + uri = f'sqlite:///{database}' + else: + uri = build_uri(scheme, database, username, password, host, port) + + return cls(uri, **kwargs) + def setup(self, uri, collection, **db_kwargs): # Setup connection to our DB: engine = create_engine(uri, **db_kwargs) diff --git a/py2store/persisters/ssh_persister.py b/py2store/persisters/ssh_persister.py index 7578b88..2d5a8cd 100644 --- a/py2store/persisters/ssh_persister.py +++ b/py2store/persisters/ssh_persister.py @@ -2,6 +2,7 @@ import stat from py2store.util import ModuleNotFoundErrorNiceMessage +from py2store.utils.uri_parsing import parse_uri, build_uri with ModuleNotFoundErrorNiceMessage(): import paramiko @@ -66,39 +67,37 @@ class SshPersister(Persister): def __init__( self, - uri, # Example: dict(url='10.1.103.201', user='stud', password='stud') + uri, collection='./py2store', encoding='utf8', + **connection_kwargs ): """ - :param uri: A dict of the following key-values: - :param str hostname: the server to connect to, REQUIRED, others are optional - :param int port: the server port to connect to - :param str username: - the username to authenticate as (defaults to the current local - username) - :param str password: - Used for password authentication; is also used for private key - decryption if ``passphrase`` is not given. - :param str passphrase: - Used for decrypting private keys. - :param .PKey pkey: an optional private key to use for authentication - :param str key_filename: - the filename, or list of filenames, of optional private key(s) - and/or certs to try for authentication - :param float timeout: - an optional timeout (in + :param uri: connection URI, formatted like "ssh://username:password@localhost:port" :param collection: root direction :param encoding: """ + parsed_uri = parse_uri(uri) + self._ssh = paramiko.SSHClient() self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self._ssh.connect(**uri) + self._ssh.connect( + hostname=parsed_uri['host'], + port=parsed_uri.get('port', 22), + username=parsed_uri.get('username'), + password=parsed_uri.get('password'), + **connection_kwargs + ) self._sftp = self._ssh.open_sftp() self._rootdir = collection self._encoding = encoding remote_mkdir(self._sftp, self._rootdir) + @classmethod + def from_kwargs(cls, host, port=22, username=None, password=None, scheme='ssh', **kwargs): + uri = build_uri(scheme, '', username, password, host, port) + return cls(uri, **kwargs) + def __getitem__(self, k): remote_file = self._sftp.file(k, mode='r') data = remote_file.read().decode(self._encoding) diff --git a/py2store/utils/uri_parsing.py b/py2store/utils/uri_parsing.py new file mode 100644 index 0000000..3145a02 --- /dev/null +++ b/py2store/utils/uri_parsing.py @@ -0,0 +1,42 @@ +from urllib.parse import urlsplit + + +def parse_uri(uri): + """ + Parses DB URI string into a dict of params. + + :param uri: string formatted as: "scheme://username:password@host:port/database" + :return: a dict with these params parsed. + """ + splitted_uri = urlsplit(uri) + + if splitted_uri.path.startswith('/'): + path = splitted_uri.path[1:] + else: + path = '' + + return { + 'scheme': splitted_uri.scheme, + 'database': path, + 'username': splitted_uri.username, + 'password': splitted_uri.password, + 'hostname': splitted_uri.hostname, + 'port': splitted_uri.port, + } + + +def build_uri( + scheme, + database='', + username=None, + password=None, + host='localhost', + port=None, +): + """ + Reverse of `parse_uri` function. + Builds a URI string from provided params. + """ + port_ = f':{port}' if port else '' + uri = f'{scheme}://{username}:{password}@{host}{port_}/{database}' + return uri diff --git a/tests/test_sql_w_sqlalchemy.py b/tests/test_sql_w_sqlalchemy.py index 02d87a7..eef4975 100644 --- a/tests/test_sql_w_sqlalchemy.py +++ b/tests/test_sql_w_sqlalchemy.py @@ -21,10 +21,20 @@ def _assert_eq(self, obj_from_db, data): assert val_from_db == data_val +class TestSQLAlchemyPersisterInitedFromKwargs(TestSQLAlchemyPersister): + db = SQLAlchemyPersister.from_kwargs( + scheme='sqlite', + database='test2.db', + collection=SQLITE_TABLE_NAME, + key_fields=list(BasePersisterTest.key.keys()), + data_fields=list(BasePersisterTest.data.keys()), + ) + + class TestSQLAlchemyTupleStore(BaseTupleStoreTest): db = SQLAlchemyTupleStore( - db_uri=SQLITE_DB_URI, - collection_name=SQLITE_TABLE_NAME, + uri=SQLITE_DB_URI, + collection=SQLITE_TABLE_NAME, key_fields=BaseTupleStoreTest.key_fields, data_fields=BaseTupleStoreTest.data_fields, ) From 156daf8d29cf605fe01c9a254a2ed1dc9bd22f7b Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 17:22:04 +0300 Subject: [PATCH 4/9] Test added for MongoPersister (update is broken). --- py2store/persisters/mongo_w_pymongo.py | 10 +++------- tests/test_mongo_w_pymongo.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 tests/test_mongo_w_pymongo.py diff --git a/py2store/persisters/mongo_w_pymongo.py b/py2store/persisters/mongo_w_pymongo.py index 384489d..a04f803 100644 --- a/py2store/persisters/mongo_w_pymongo.py +++ b/py2store/persisters/mongo_w_pymongo.py @@ -63,14 +63,10 @@ def __init__( key_fields=('_id',), data_fields=None, ): - # db_name = uri.pop('db_name') - uri, db_name = uri.rsplit('/', 1) - uri_parsed = parse_uri(uri) - - self._db_name = db_name - self._mongo_client = MongoClient(**uri_parsed) + uri, self._db_name = uri.rsplit('/', 1) + self._mongo_client = MongoClient(uri) self._collection_name = collection - self._mgc = self._mongo_client[db_name][collection] + self._mgc = self._mongo_client[self._db_name][collection] if isinstance(key_fields, str): key_fields = (key_fields,) diff --git a/tests/test_mongo_w_pymongo.py b/tests/test_mongo_w_pymongo.py new file mode 100644 index 0000000..ae28b74 --- /dev/null +++ b/tests/test_mongo_w_pymongo.py @@ -0,0 +1,26 @@ +""" +How to run a test MongoDB instance locally with a Docker container: + +docker rm -f mongo-test && \ +docker run -d -p 27017:27017 --name mongo-test mongo && \ +sleep 10 && \ +pytest tests/test_mongo_w_pymongo.py +""" +from py2store.persisters.mongo_w_pymongo import MongoPersister + +from tests.base_test import BasePersisterTest + +URI = 'mongodb://localhost:27017/test_db' + + +class TestMongoPersister(BasePersisterTest): + db = MongoPersister( + URI, + key_fields=list(BasePersisterTest.key.keys()), + ) + + def _assert_eq(self, obj_from_db, data): + """ Custom comparison by attributes, since persister returns dicts. """ + for data_key, data_val in data.items(): + val_from_db = obj_from_db[data_key] + assert val_from_db == data_val From a4710a6814bb1169c3502798a6c8c37e7d3e6529 Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 17:25:32 +0300 Subject: [PATCH 5/9] Minor ArangoDB Test docstring change (to be alike Mongo's). --- tests/test_arangodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_arangodb.py b/tests/test_arangodb.py index 044a258..bffb728 100644 --- a/tests/test_arangodb.py +++ b/tests/test_arangodb.py @@ -1,8 +1,8 @@ """ How to run a test ArangoDB instance locally with a Docker container: -docker rm -f arangodb-instance && \ -docker run -e ARANGO_ROOT_PASSWORD=somepassword -p 8529:8529 -d --name arangodb-instance arangodb && \ +docker rm -f arangodb-test && \ +docker run -e ARANGO_ROOT_PASSWORD=somepassword -p 8529:8529 -d --name arangodb-test arangodb && \ sleep 10 && \ pytest tests/test_arangodb.py """ From 06d10fba3ce2749e530fc0ca99d743ad536aa593 Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 18:09:50 +0300 Subject: [PATCH 6/9] Added CouchDB tests. --- py2store/persisters/couchdb_w_couchdb.py | 4 +++- tests/test_arangodb.py | 6 +++--- tests/test_couchdb_w_couchdb.py | 20 ++++++++++++++++++++ tests/test_mongo_w_pymongo.py | 6 +++--- 4 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 tests/test_couchdb_w_couchdb.py diff --git a/py2store/persisters/couchdb_w_couchdb.py b/py2store/persisters/couchdb_w_couchdb.py index 9b20b5e..04a1c77 100644 --- a/py2store/persisters/couchdb_w_couchdb.py +++ b/py2store/persisters/couchdb_w_couchdb.py @@ -1,9 +1,11 @@ + from py2store.base import Persister from py2store.util import ModuleNotFoundErrorNiceMessage from py2store.utils.uri_parsing import build_uri with ModuleNotFoundErrorNiceMessage(): from couchdb import Server + from couchdb.client import DEFAULT_BASE_URL class CouchDbPersister(Persister): @@ -66,7 +68,7 @@ def clear(self): def __init__( self, - uri, # Example: "https://username:password@example.com:5984/" + uri=DEFAULT_BASE_URL, collection='py2store', key_fields=('_id',), couchdb_client_kwargs=None diff --git a/tests/test_arangodb.py b/tests/test_arangodb.py index bffb728..68cdca9 100644 --- a/tests/test_arangodb.py +++ b/tests/test_arangodb.py @@ -1,9 +1,9 @@ """ How to run a test ArangoDB instance locally with a Docker container: -docker rm -f arangodb-test && \ -docker run -e ARANGO_ROOT_PASSWORD=somepassword -p 8529:8529 -d --name arangodb-test arangodb && \ -sleep 10 && \ +docker rm -f arangodb-test +docker run -e ARANGO_ROOT_PASSWORD=somepassword -p 8529:8529 -d --name arangodb-test arangodb +sleep 10 pytest tests/test_arangodb.py """ from py2store.persisters._arangodb_in_progress import ArangoDbPersister diff --git a/tests/test_couchdb_w_couchdb.py b/tests/test_couchdb_w_couchdb.py new file mode 100644 index 0000000..f7d4509 --- /dev/null +++ b/tests/test_couchdb_w_couchdb.py @@ -0,0 +1,20 @@ +""" +How to run a test CouchDB instance locally with a Docker container: + +docker rm -f couchdb-test +docker run -d -p 5984:5984 --name couchdb-test couchdb +sleep 10 +pytest tests/test_couchdb_w_couchdb.py +""" +from py2store.persisters.couchdb_w_couchdb import CouchDbPersister + +from tests.base_test import BasePersisterTest + +URI = 'http://localhost:5984' + + +class TestCouchDBPersister(BasePersisterTest): + db = CouchDbPersister( + URI, + key_fields=list(BasePersisterTest.key.keys()), + ) diff --git a/tests/test_mongo_w_pymongo.py b/tests/test_mongo_w_pymongo.py index ae28b74..332897e 100644 --- a/tests/test_mongo_w_pymongo.py +++ b/tests/test_mongo_w_pymongo.py @@ -1,9 +1,9 @@ """ How to run a test MongoDB instance locally with a Docker container: -docker rm -f mongo-test && \ -docker run -d -p 27017:27017 --name mongo-test mongo && \ -sleep 10 && \ +docker rm -f mongo-test +docker run -d -p 27017:27017 --name mongo-test mongo +sleep 10 pytest tests/test_mongo_w_pymongo.py """ from py2store.persisters.mongo_w_pymongo import MongoPersister From d1d3a51306ac90c209c962257af40ff84ad81f03 Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 18:35:47 +0300 Subject: [PATCH 7/9] Added FTP read tests. --- py2store/persisters/ftp_persister.py | 2 +- tests/test_arangodb.py | 2 +- tests/test_couchdb_w_couchdb.py | 2 +- tests/test_ftp.py | 12 ++++++++++++ tests/test_mongo_w_pymongo.py | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 tests/test_ftp.py diff --git a/py2store/persisters/ftp_persister.py b/py2store/persisters/ftp_persister.py index cf7562b..dd3ec69 100644 --- a/py2store/persisters/ftp_persister.py +++ b/py2store/persisters/ftp_persister.py @@ -70,7 +70,7 @@ def __init__( uri_parsed = parse_uri(uri) self._ftp = FTP( - host=uri_parsed['host'], + host=uri_parsed['hostname'], user=uri_parsed['username'], passwd=uri_parsed['password'], ) diff --git a/tests/test_arangodb.py b/tests/test_arangodb.py index 68cdca9..6a8614e 100644 --- a/tests/test_arangodb.py +++ b/tests/test_arangodb.py @@ -1,5 +1,5 @@ """ -How to run a test ArangoDB instance locally with a Docker container: +How to run a test of ArangoDB instance locally with a Docker container: docker rm -f arangodb-test docker run -e ARANGO_ROOT_PASSWORD=somepassword -p 8529:8529 -d --name arangodb-test arangodb diff --git a/tests/test_couchdb_w_couchdb.py b/tests/test_couchdb_w_couchdb.py index f7d4509..7f666c2 100644 --- a/tests/test_couchdb_w_couchdb.py +++ b/tests/test_couchdb_w_couchdb.py @@ -1,5 +1,5 @@ """ -How to run a test CouchDB instance locally with a Docker container: +How to run a test of CouchDB instance locally with a Docker container: docker rm -f couchdb-test docker run -d -p 5984:5984 --name couchdb-test couchdb diff --git a/tests/test_ftp.py b/tests/test_ftp.py new file mode 100644 index 0000000..8356bb3 --- /dev/null +++ b/tests/test_ftp.py @@ -0,0 +1,12 @@ +from py2store.persisters.ftp_persister import FtpPersister + +URI = 'ftp://anonymous:anonymous@speedtest.tele2.net/' +ROOT_DIR = '/' +FILE_TO_READ = '1KB.zip' + + +class TestFtpPersister: + db = FtpPersister(URI, collection=ROOT_DIR) + + def test_read_file(self): + assert self.db[FILE_TO_READ] diff --git a/tests/test_mongo_w_pymongo.py b/tests/test_mongo_w_pymongo.py index 332897e..1838557 100644 --- a/tests/test_mongo_w_pymongo.py +++ b/tests/test_mongo_w_pymongo.py @@ -1,5 +1,5 @@ """ -How to run a test MongoDB instance locally with a Docker container: +How to run a test of MongoDB instance locally with a Docker container: docker rm -f mongo-test docker run -d -p 27017:27017 --name mongo-test mongo From 90eb4b22cb409534bccc9d875b2976369b7eca42 Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 18:59:34 +0300 Subject: [PATCH 8/9] Added MSSQL broken tests (yet). --- py2store/persisters/sql_w_odbc.py | 2 +- tests/_test_sql_w_odbc_in_progress.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/_test_sql_w_odbc_in_progress.py diff --git a/py2store/persisters/sql_w_odbc.py b/py2store/persisters/sql_w_odbc.py index 648f0eb..b61b0bb 100644 --- a/py2store/persisters/sql_w_odbc.py +++ b/py2store/persisters/sql_w_odbc.py @@ -28,7 +28,7 @@ def __init__(self, uri, collection='py2store_default_table', primary_key='id', d uri_parsed = parse_uri(uri) self._sql_server_client = pyodbc.connect( 'DRIVER={{ODBC Driver 17 for SQL Server}};' - 'SERVER={scheme}:{host},{port};' + 'SERVER={scheme}:{hostname},{port};' 'DATABASE={database};' 'UID={username};' 'PWD={password}' diff --git a/tests/_test_sql_w_odbc_in_progress.py b/tests/_test_sql_w_odbc_in_progress.py new file mode 100644 index 0000000..1f31c19 --- /dev/null +++ b/tests/_test_sql_w_odbc_in_progress.py @@ -0,0 +1,23 @@ +""" +How to run a test of MSSQL instance locally with a Docker container: + +docker rm -f mssql-test +docker run -e ACCEPT_EULA=Y -e SA_PASSWORD=password -p 1433:1433 -d --name mssql-test mcr.microsoft.com/mssql/server:2017-CU8-ubuntu +sleep 10 +pytest tests/_test_sql_w_odbc_in_progress.py +""" +from py2store.persisters.sql_w_odbc import SQLServerPersister + +from tests.base_test import BasePersisterTest + +URI = 'tcp://localhost:1433/test_db' + + +class TestSQLServerPersister(BasePersisterTest): + db = SQLServerPersister(URI) + + # def _assert_eq(self, obj_from_db, data): + # """ Custom comparison by attributes, since persister returns dicts. """ + # for data_key, data_val in data.items(): + # val_from_db = obj_from_db[data_key] + # assert val_from_db == data_val From 58d7085aeb34d65c5c2cd9baee278b44f6e4c5d3 Mon Sep 17 00:00:00 2001 From: Mikhail Savushkin Date: Fri, 1 Nov 2019 19:18:28 +0300 Subject: [PATCH 9/9] SSHPersister fix. --- py2store/persisters/ssh_persister.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py2store/persisters/ssh_persister.py b/py2store/persisters/ssh_persister.py index 2d5a8cd..230380c 100644 --- a/py2store/persisters/ssh_persister.py +++ b/py2store/persisters/ssh_persister.py @@ -82,7 +82,7 @@ def __init__( self._ssh = paramiko.SSHClient() self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self._ssh.connect( - hostname=parsed_uri['host'], + hostname=parsed_uri['hostname'], port=parsed_uri.get('port', 22), username=parsed_uri.get('username'), password=parsed_uri.get('password'),