From 8ec9d88406e0718e524f6ce1c379f6b783ace8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Sun, 6 Oct 2019 16:17:03 +0200 Subject: [PATCH] remove all traces of firebase, it is put in a designated PR --- docs/developers_guide.rst | 22 -- docs/getting_started.rst | 23 +- docs/introduction.rst | 6 +- expipe/backends/__init__.py | 2 +- expipe/backends/firebase.py | 443 ----------------------------------- expipe/tests/conftest.py | 4 - expipe/tests/mock_backend.py | 77 ------ 7 files changed, 6 insertions(+), 571 deletions(-) delete mode 100644 expipe/backends/firebase.py delete mode 100644 expipe/tests/mock_backend.py diff --git a/docs/developers_guide.rst b/docs/developers_guide.rst index 7a3aa5e..d89b196 100644 --- a/docs/developers_guide.rst +++ b/docs/developers_guide.rst @@ -40,27 +40,5 @@ You can then push your changes to your online repository on GitHub:: Once you think your changes are ready to be included in the main Neo repository, open a pull request on GitHub (see https://help.github.com/articles/using-pull-requests). -Testing -------- - -**WARNING**: Tests can be destructive and have to be run on a different server. -Use `expipe-debug.firebaseio.com` for testing. - -A separate config file needs to be created if you want to run tests. The config -file needs to be placed in `~/.config/expipe/test-config.yaml` and must have -the key `allow_tests: true`. Here is an example: - -.. code-block:: yaml - - allow_tests: true - data_path: '/tmp/expipe-data' - firebase: - email: - password: - config: - apiKey: AIzaSyBnbsraKxrO8zv1qVZeAvJR4fEWzExQhOM - authDomain: expipe-debug.firebaseapp.com - databaseURL: https://expipe-debug.firebaseio.com - storageBucket: expipe-debug.appspot.com .. _GitHub: http://github.com diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 712b3b8..d7ef588 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -15,7 +15,7 @@ Create a new project if it does not exist with ``require_project``: >>> project = expipe.require_project("test") The default backend for Expipe uses the filesystem. -In this case, the above command will create a folder named `test` in your current +In this case, the above command will create a folder named `test` in your current working directory. Other backends will create a backend-specific project in its database. @@ -101,7 +101,7 @@ To further retrieve and edit the values of a module, you can use `module.to_dict From template to module ----------------------- -To upload a template you can write it in ``json`` or as a ``dict`` and use +To upload a template you can write it as a ``dict`` and use ``require_template``. .. doctest:: @@ -113,10 +113,6 @@ To upload a template you can write it in ``json`` or as a ``dict`` and use >>> expipe.require_template(template='hardware_daq', ... contents=daq_contents) -Contents can also be a ``.json`` file:: - - expipe.require_template(template='hardware_daq', - contents='daq_contents.json') In order to use a template and add it as a module to an `action` use ``action.require_module``: @@ -137,21 +133,6 @@ and values use ``to_dict``: >>> print(daq_dict.values()) odict_values([{'definition': 'The number of input channels of the DAQ-device.', 'value': '64'}]) -You may also view the module as ``.json`` by using the command ``to_json``: - -.. doctest:: - - >>> daq.to_json() - Saving module "hardware_daq" to "hardware_daq.json" - -To furter change its values and upload them to Firebase: - -.. doctest:: - - >>> daq_dict['gain'] = {'value': 20} - >>> daq = action.require_module(name='hardware_daq', contents=daq_dict, - ... overwrite=True) - Messages -------- diff --git a/docs/introduction.rst b/docs/introduction.rst index 55339fa..c7ac356 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -14,9 +14,9 @@ To ease this organization we introduce a tool for the experiment pipeline named * support high throughput data analysis. * support multiple types of large-scale data sets. -To this end we use the flexible `Firebase `_ -NoSQL type database to store metadata. This way of storing metadata consist of -assigning key value pairs in a `json `_ type format. +To this end we use the flexible filesystem as a +NoSQL type database to store data and metadata. This way of storing metadata consist of +assigning key value pairs. During an experiment metadata can be automatically added by user specific `templates`. Templates are prefilled key value pairs describing all aspects diff --git a/expipe/backends/__init__.py b/expipe/backends/__init__.py index ea6c51a..8b13789 100644 --- a/expipe/backends/__init__.py +++ b/expipe/backends/__init__.py @@ -1 +1 @@ -from .firebase import FirebaseBackend + diff --git a/expipe/backends/firebase.py b/expipe/backends/firebase.py deleted file mode 100644 index ba8596c..0000000 --- a/expipe/backends/firebase.py +++ /dev/null @@ -1,443 +0,0 @@ -from ..backend import * -import requests -from ..core import * -import datetime as dt -import expipe -try: - import ruamel.yaml as yaml -except ImportError: - import ruamel_yaml as yaml - -def deep_verification(default, current, path=""): - for key in default: - next_path = key - if path: - next_path = path + "." + key - - if key not in current: - print("WARNING: '{}' not found in settings.".format(next_path), - "Please rerun expipe.configure().") - else: - if isinstance(default[key], dict): - if not isinstance(current[key], dict): - print("WARNING: Expected '{}' to be dict in settings.".format(next_path), - "Please rerun expipe.configure().") - else: - deep_verification(default[key], current[key], path=next_path) - - -def configure(config_name, data_path, email, password, url_prefix, api_key): - """ - The configure function creates a configuration file if it does not yet exist. - Ask your expipe administrator about the correct values for the parameters. - - Parameters - ---------- - data_path : - path to where data files should be stored - email : - user email on Firebase server - password : - user password on Firebase server (WARNING: Will be stored in plain text!) - url_prefix: - prefix of Firebase server URL (https://.firebaseio.com) - api_key: - Firebase API key - """ - - config_dir = pathlib.home() / '.config' / 'expipe' / 'firebase' - config_dir.mkdir(exist_ok=True) - config_file = (config_dir / config_name).with_suffix(".yaml") - - current_settings = {} - with open(config_file) as f: - current_settings = yaml.safe_load(f) - - current_settings.update({ - "data_path": data_path, - "firebase": { - "email": email, - "password": password, - "config": { - "apiKey": api_key, - "authDomain": "{}.firebaseapp.com".format(url_prefix), - "databaseURL": "https://{}.firebaseio.com".format(url_prefix), - "storageBucket": "{}.appspot.com".format(url_prefix) - } - } - }) - - with open(config_file) as f: - yaml.dump(current_settings, f, default_flow_style=False) - - -def convert_from_firebase(value): - """ - Converts quantities back from dictionary - """ - result = value - if isinstance(value, dict): - if "_force_dict" in value: - del value["_force_dict"] - if 'units' in value and "value" in value: - value['unit'] = value['units'] - del(value['units']) - if "unit" in value and "value" in value: - if "uncertainty" in value: - try: - result = pq.UncertainQuantity(value["value"], - value["unit"], - value["uncertainty"]) - except Exception: - pass - else: - try: - result = pq.Quantity(value["value"], value["unit"]) - except Exception: - pass - else: - try: - for key, value in result.items(): - result[key] = convert_from_firebase(value) - except AttributeError: - pass - if isinstance(result, str): - if result == 'NaN': - result = np.nan - elif isinstance(result, list): - result = [v if v != 'NaN' else np.nan for v in result] - return result - - -def convert_to_firebase(value): - """ - Converts quantities to dictionary - """ - if isinstance(value, dict): - if all(isinstance(key, int) or (isinstance(key, str) and key.isnumeric()) for key in value): - value["_force_dict"] = True - if isinstance(value, np.ndarray) and not isinstance(value, pq.Quantity): - if value.ndim >= 1: - value = value.tolist() - if isinstance(value, list): - value = [convert_to_firebase(val) for val in value] - - result = value - - if isinstance(value, pq.Quantity): - try: - val = ['NaN' if np.isnan(r) else r for r in value.magnitude] - except TypeError: - val = value.magnitude.tolist() - result = {"value": val, - "unit": value.dimensionality.string} - if isinstance(value, pq.UncertainQuantity): - assert(value.dimensionality == value.uncertainty.dimensionality) - result["uncertainty"] = value.uncertainty.magnitude.tolist() - elif isinstance(value, np.integer): - result = int(value) - elif isinstance(value, np.float): - result = float(value) - else: - # try if dictionary like objects can be converted if not return the - # original object - # Note, this might fail if .items() returns a strange combination of - # objects - try: - new_result = {} - for key, val in value.items(): - new_key = convert_to_firebase(key) - new_result[new_key] = convert_to_firebase(val) - result = new_result - except AttributeError: - pass - try: - if not isinstance(value, list) and np.isnan(result): - result = 'NaN' - except TypeError: - pass - return result - - -class FirebaseObjectManager(AbstractObjectManager): - def __init__(self, config, path, path_prefix, object_type, backend_type): - self.config = config - self.path = path - self.path_prefix = path_prefix - self._db = FirebaseObject(config, path) - self._object_type = object_type - self._backend_type = backend_type - - def __getitem__(self, name): - if not self._db.exists(name): - raise KeyError("Action '{}' does not exist".format(name)) - if self.path_prefix is not None: - prefixed_name = "/".join([self.path_prefix, name]) - else: - prefixed_name = name - - full_path = "/".join([self.path, name]) - - return self._object_type(name, self._backend_type(self.config, full_path, prefixed_name)) - - def __iter__(self): - keys = self._db.get(shallow=True) or [] - for key in keys: - yield key - - def __len__(self): - keys = self._db.get(shallow=True) or [] - return len(keys) - - def __contains__(self, name): - return name in (self._db.get(shallow=True) or []) - - def __setitem__(self, name, value): - self._db.set(name, value) - - -class FirebaseBackend(AbstractBackend): - def __init__(self, config): - self.config = config - self.projects = FirebaseObjectManager(config, "projects", None, Project, FirebaseProject) - - def exists(self, name): - return name in self.projects - - def get_project(self, name): - return Project(name, FirebaseProject(self.config, None, name)) - - def create_project(self, name, contents): - self.projects[name] = contents - - def delete_project(self, name, remove_all_children=False): - raise NotImplementedError("Cannot delete firebase projects") - - -class FirebaseProject: - def __init__(self, config, path, name): - self._attribute_manager = FirebaseObject(config, path) - self._action_manager = FirebaseObjectManager( - config, - "/".join(["actions", name]), name, Action, FirebaseAction) - self._entities_manager = FirebaseObjectManager( - config, - "/".join(["entities", name]), name, Entity, FirebaseEntity) - self._templates_manager = FirebaseObjectManager( - config, - "/".join(["templates", name]), name, Template, FirebaseTemplate) - self._module_manager = FirebaseObjectManager( - config, - "/".join(["project_modules", name]), name, Template, FirebaseTemplate) - - @property - def actions(self): - return self._action_manager - - @property - def entities(self): - return self._entities_manager - - @property - def templates(self): - return self._templates_manager - - @property - def attributes(self): - return self._attribute_manager - - @property - def modules(self): - return self._module_manager - - -class FirebaseAction: - def __init__(self, config, path, name): - self._attribute_manager = FirebaseObject(config, path) - self._module_manager = FirebaseObjectManager( - config, - "/".join(["action_modules", name]), path, Module, FirebaseModule) - self._message_manager = FirebaseObjectManager( - config, - "/".join(["action_messages", name]), path, Message, FirebaseMessage) - - @property - def modules(self): - return self._module_manager - - @property - def attributes(self): - return self._attribute_manager - - @property - def messages(self): - return self._message_manager - - -class FirebaseMessage: - def __init__(self, config, path, name): - self._content_manager = FirebaseObject(config, path) - - @property - def contents(self): - return self._content_manager - - -class FirebaseEntity: - def __init__(self, config, path, name): - self._attribute_manager = FirebaseObject(config, path) - self._message_manager = FirebaseObjectManager(config, "/".join(["entity_messages", name]), path, Message, FirebaseMessage) - self._module_manager = FirebaseObjectManager(config, "/".join(["entity_modules", name]), path, Message, FirebaseModule) - - @property - def modules(self): - return self._module_manager - - @property - def attributes(self): - return self._attribute_manager - - @property - def messages(self): - return self._message_manager - -class FirebaseModule: - def __init__(self, config, path, name): - self._content_manager = FirebaseObject(config, path) - - @property - def contents(self): - return self._content_manager - - -class FirebaseTemplate: - def __init__(self, config, path): - self._content_manager = FirebaseObject(config, path) - - def content_manager(self): - return self._content_manager - - -class FirebaseObject(AbstractObject): - def __init__(self, config, path): - super(FirebaseObject, self).__init__() - self.id_token = None - self.refresh_token = None - self.token_expiration = dt.datetime.now() - self.path = path - self.config = config - - def ensure_auth(self): - current_time = dt.datetime.now() - api_key = self.config["firebase"]["config"]["apiKey"] - - if self.id_token is not None and self.refresh_token is not None: - if current_time + dt.timedelta(0, 10) < self.token_expiration and False: - return - auth_url = "https://securetoken.googleapis.com/v1/token?key={}".format(api_key) - auth_data = { - "grant_type": "refresh_token", - "refresh_token": self.refresh_token - } - - response = requests.post(auth_url, json=auth_data) - value = response.json() - assert(response.status_code == 200) - assert("errors" not in value) - self.id_token = value["id_token"] - self.refresh_token = value["refresh_token"] - return - - auth_url = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key={}".format(api_key) - auth_data = { - "email": self.config["firebase"]["email"], - "password": self.config["firebase"]["password"], - "returnSecureToken": True - } - response = requests.post(auth_url, json=auth_data) - assert(response.status_code == 200) - value = response.json() - assert("errors" not in value) - self.refresh_token = value["refreshToken"] - self.id_token = value["idToken"] - self.token_expiration = current_time + dt.timedelta(0, int(value["expiresIn"])) - - def build_url(self, name=None): - if name is None: - full_path = self.path - else: - full_path = "/".join([self.path, name]) - database_url = self.config["firebase"]["config"]["databaseURL"] - result = "{database_url}/{name}.json?auth={id_token}".format( - database_url=database_url, - name=full_path, - id_token=self.id_token - ) - return result - - def exists(self, name=None): - self.ensure_auth() - value = self.get(name, shallow=True) - if value is not None: - return True - else: - return False - - def get(self, name=None, shallow=False): - self.ensure_auth() - url = self.build_url(name) - if shallow: - url += "&shallow=true" - response = requests.get(url) - assert(response.status_code == 200) - value = response.json() - if value is None: - return value - assert("errors" not in value) - value = convert_from_firebase(value) - return value - - # def get_keys(self, name=None): - # return self.get(name, shallow=True) - - def set(self, name=None, value=None): - self.ensure_auth() - url = self.build_url(name) - if value is None: - value = name - response = requests.put(url, json=value) - assert(response.status_code == 200) - value = response.json() - if value is not None: - assert("errors" not in value) - - def push(self, name=None, value=None): - self.ensure_auth() - url = self.build_url(name) - if value is None: - value = name - response = requests.post(url, json=value) - assert(response.status_code == 200) - value = response.json() - if value is None: - return value - assert("errors" not in value) - return value - - def delete(self, name): - self.set(name, {}) - - def update(self, name, value=None): - self.ensure_auth() - url = self.build_url(name) - if value is None: - value = name - value = convert_to_firebase(value) - response = requests.patch(url, json=value) - assert(response.status_code == 200) - value = response.json() - if value is None: - return value - assert("errors" not in value) - value = convert_from_firebase(value) - return value diff --git a/expipe/tests/conftest.py b/expipe/tests/conftest.py index 48c39fd..0921185 100644 --- a/expipe/tests/conftest.py +++ b/expipe/tests/conftest.py @@ -28,10 +28,6 @@ def pytest_configure(): pytest.TEMPLATE_ID = TEMPLATE_ID -def pytest_addoption(parser): - parser.addoption("--firebase", action="store_true", default=False) - - @pytest.fixture(scope='function') def project_path(): if os.path.exists(TESTDIR): diff --git a/expipe/tests/mock_backend.py b/expipe/tests/mock_backend.py deleted file mode 100644 index 3b4133b..0000000 --- a/expipe/tests/mock_backend.py +++ /dev/null @@ -1,77 +0,0 @@ -import dpath.util -import expipe -import copy - - -def create_mock_backend(data=None): - data = {} if data is None else data - - class MockBackend(expipe.core.AbstractBackend): - def __init__(self, path): - super(MockBackend, self).__init__( - path=path - ) - self.data = data - - def exists(self, name=None): - value = self.get(name) - if value is not None: - return True - else: - return False - - def get(self, name=None, shallow=False): - try: - if name is None: - value = dpath.util.get(glob=self.path, obj=self.data) - else: - if not isinstance(name, str): - raise TypeError('Expected "str", not "{}"'.format(type(name))) - value = dpath.util.get(glob=self.path + "/" + name, obj=self.data) - value = copy.deepcopy(value) - except KeyError: - value = None - value = expipe.core.convert_from_firebase(value) - return value - - # def get_keys(self, name=None): - # if name is None: - # value = dpath.util.get(glob=self.path, obj=self.data) - # else: - # if not isinstance(name, str): - # raise TypeError('Expected "str", not "{}"'.format(type(name))) - # value = dpath.util.get(glob=self.path + "/" + name, obj=self.data) - # return value.keys() - - def set(self, name, value=None): - if value is None: - value = name - value = expipe.core.convert_to_firebase(value) - dpath.util.new(path=self.path, obj=self.data, value=value) - else: - value = expipe.core.convert_to_firebase(value) - dpath.util.new(path=self.path + "/" + str(name), obj=self.data, value=value) - - def push(self, value=None): - import numpy as np - name = str(np.random.random()) - self.set(name=name, value=value) - return {"name": name} - - def delete(self, name): - dpath.util.delete(glob=self.path + "/" + str(name), obj=self.data) - - def update(self, name, value=None): - if value is None: - value = name - value = expipe.core.convert_to_firebase(value) - dpath.util.new(path=self.path, obj=self.data, value=value) - # db.child(self.path).update(value, user["idToken"]) - else: - value = expipe.core.convert_to_firebase(value) - dpath.util.new(path=self.path + "/" + name, obj=self.data, value=value) - # db.child(self.path).child(name).update(value, user["idToken"]) - value = expipe.core.convert_from_firebase(value) - return value - - return MockBackend