diff --git a/.gitignore b/.gitignore index b6888d5..8fe2b90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*~ *.pyc .venv/ .venv3/ diff --git a/Makefile b/Makefile index 984dda1..68bbfca 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -APT_PREREQS=python-dev python3-dev python-virtualenv +APT_PREREQS=python3-dev python-virtualenv PROJECT=pgsql TESTS=tests/ @@ -12,20 +12,15 @@ all: .PHONY: clean clean: find . -name '*.pyc' -delete - rm -rf .venv + find . -name '__pycache__' -delete + find . -name '*~' -delete rm -rf .venv3 - rm -rf docs/_build + rm -rf docs/build .PHONY: docclean docclean: -rm -rf docs/_build -.venv: - @echo Processing apt package prereqs - @for i in $(APT_PREREQS); do dpkg -l | grep -w $$i >/dev/null || sudo apt-get install -y $$i; done - virtualenv .venv - .venv/bin/pip install -IUr test_requirements.txt - .venv3: @echo Processing apt package prereqs @for i in $(APT_PREREQS); do dpkg -l | grep -w $$i >/dev/null || sudo apt-get install -y $$i; done @@ -33,12 +28,12 @@ docclean: .venv3/bin/pip install -IUr test_requirements.txt .PHONY: lint -lint: .venv .venv3 - @echo Checking for Python syntax... - .venv/bin/flake8 --max-line-length=120 $(PROJECT) $(TESTS) \ - && echo Py2 OK - .venv3/bin/flake8 --max-line-length=120 $(PROJECT) $(TESTS) \ - && echo Py3 OK +# lint: .venv3 +# @echo Checking for Python syntax... +# .venv3/bin/flake8 $(PROJECT) $(TESTS) \ +# && echo Py3 OK +lint: + flake8 requires.py # Note we don't even attempt to run tests if lint isn't passing. .PHONY: test @@ -55,11 +50,5 @@ test3: .venv3 .venv3/bin/nosetests -s --nologcapture tests/ .PHONY: docs -docs: .venv - - [ -z "`.venv/bin/pip list | grep -i 'sphinx '`" ] && .venv/bin/pip install sphinx - - [ -z "`.venv/bin/pip list | grep -i sphinx-pypi-upload`" ] && .venv/bin/pip install sphinx-pypi-upload - # If sphinx is installed on the system, pip installing into the venv does not - # put the binaries into .venv/bin. Test for and use the .venv binary if it's - # there; otherwise, we probably have a system sphinx in /usr/bin, so use that. - SPHINX=$$(test -x .venv/bin/sphinx-build && echo \"../.venv/bin/sphinx-build\" || echo \"../.venv/bin/python /usr/bin/sphinx-build\"); \ - cd docs && make html SPHINXBUILD=$$SPHINX && cd - +docs: .venv3 + make -C docs html SPHINXBUILD=../.venv3/bin/sphinx-build diff --git a/common/__init__.py b/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/copyright b/copyright index e900b97..b78610c 100644 --- a/copyright +++ b/copyright @@ -1,16 +1,16 @@ Format: http://dep.debian.net/deps/dep5/ Files: * -Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. -License: Apache License 2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Copyright: Copyright 2015-2016, Canonical Ltd. +License: GPL-3 + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. . - http://www.apache.org/licenses/LICENSE-2.0 + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranties of + MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU General Public License for more details. . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + You should have received a copy of the GNU General Public License + along with this program. If not, see . diff --git a/docs/source/index.rst b/docs/source/index.rst index 9c09712..af7d496 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,6 @@ Contents: :maxdepth: 2 requires - provides Indices and tables diff --git a/docs/source/provides.rst b/docs/source/provides.rst deleted file mode 100644 index 6c1a49d..0000000 --- a/docs/source/provides.rst +++ /dev/null @@ -1,65 +0,0 @@ -Provides: PostgreSQL -==================== - -Example Usage -------------- - -This is what a charm using this relation would look like: - -.. code-block:: python - - # in the postgres charm: - from charmhelpers.core import hookenv # noqa - from charmhelpers.core import unitdata - from charmhelpers.core.reactive import when - from common import ( - user_name, - create_user, - reset_user_roles, - ensure_database, - get_service_port, - ) - - - @when('db.roles.requested') - def update_roles(pgsql): - for service, roles in pgsql.requested_roles(): - user = user_name(pgsql.relation_name(), service) - reset_user_roles(user, roles) - pgsql.ack_roles(service, roles) - - - @when('db.database.requested') - def provide_database(pgsql): - for service, database in pgsql.requested_databases(): - if not database: - database = service - roles = pgsql.requested_roles(service) - - user = user_name(pgsql.relation_name(), service) # generate username - password = create_user(user) # get-or-create user - schema_user = "{}_schema".format(user) - schema_password = create_user(schema_user) - - reset_user_roles(user, roles) - ensure_database(user, schema_user, database) - - pgsql.provide_database( - service=service, - host=hookenv.unit_private_ip(), - port=get_service_port(), - database=database, - state=unitdata.kv().get('pgsql.state'), # master, hot standby, standalone - user=user, - password=password, - schema_user=schema_user, - schema_password=schema_password, - ) - - -Reference ---------- - -.. autoclass:: - provides.PostgreSQL - :members: diff --git a/docs/source/requires.rst b/docs/source/requires.rst index fc4caca..7aec720 100644 --- a/docs/source/requires.rst +++ b/docs/source/requires.rst @@ -15,23 +15,22 @@ This is what a charm using this relation would look like: from charmhelpers.core.reactive import set_state from charmhelpers.core.reactive import remove_state - @hook('db-relation-joined') + @when('db.connected') def request_db(pgsql): - pgsql.change_database_name('mydb') - pgsql.request_roles('myrole', 'otherrole') + pgsql.set_database('mydb') - @hook('config-changed') + @when('config.changed') def check_admin_pass(): - admin_pass = hookenv.config('admin-pass') + admin_pass = hookenv.config()['admin-pass'] if admin_pass: set_state('admin-pass') else: remove_state('admin-pass') - @when('db.database.available', 'admin-pass') + @when('db.master.available', 'admin-pass') def render_config(pgsql): render_template('app-config.j2', '/etc/app.conf', { - 'db_conn': pgsql.connection_string(), + 'db_conn': pgsql.master, 'admin_pass': hookenv.config('admin-pass'), }) @@ -42,6 +41,13 @@ This is what a charm using this relation would look like: Reference --------- +.. autoclass:: + requires.ConnectionString + :members: + +.. autoclass:: + requires.ConnectionStrings + :members: .. autoclass:: requires.PostgreSQLClient diff --git a/interface.yaml b/interface.yaml index 647e367..029aaa2 100644 --- a/interface.yaml +++ b/interface.yaml @@ -1,3 +1,3 @@ name: pgsql -summary: Interface for relating to PostgreSQL +summary: PostgreSQL client interface maintainer: '"Cory Johns" ' diff --git a/peer.py b/peer.py deleted file mode 100644 index e69de29..0000000 diff --git a/provides.py b/provides.py deleted file mode 100644 index d9a8d7b..0000000 --- a/provides.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core import hookenv -from charms.reactive import RelationBase -from charms.reactive import scopes -from charms.reactive import hook -from charms.reactive import not_unless - - -class PostgreSQL(RelationBase): - # We expect multiple, separate services to be related, but all units of a - # given service will share the same database name and connection info. - # Thus, we use SERVICE scope and will have one converstaion per service. - scope = scopes.SERVICE - - @hook('{provides:pgsql}-relation-{joined,changed}') - def joined_changed(self): - """ - Handles the relation-joined and relation-changed hook. - - Depending on the state of the conversation, this can trigger one - of the following states: - - * ``{relation_name}.database.requested`` This state will be activated if - the remote service has requested a different database name than the - one it has been provided. This state should be resolved by calling - :meth:`provide_database`. See also :meth:`requested_databases`. - - * ``{relation_name}.roles.requested`` This state will be activated if - the remote service has requested a specific set of roles for its user. - This state should be resolved by calling :meth:`ack_roles`. See also - :meth:`requrested_roles`. - """ - service = hookenv.remote_service() - conversation = self.conversation() - - if self.previous_database(service) != self.requested_database(service): - conversation.set_state('{relation_name}.database.requested') - - if self.previous_roles(service) != self.requested_roles(service): - conversation.set_state('{relation_name}.roles.requested') - - @not_unless('{provides:pgsql}.database.requested') - def provide_database(self, service, host, port, database, user, password, schema_user, schema_password, state): - """ - Provide a database to a requesting service. - - :param str service: The service which requested the database, as - returned by :meth:`~provides.PostgreSQL.requested_databases`. - :param str host: The host where the database can be reached (e.g., - the charm's private or public-address). - :param int port: The port where the database can be reached. - :param str database: The name of the database being provided. - :param str user: The username to be used to access the database. - :param str password: The password to be used to access the database. - :param str schema_user: The username to be used to admin the database. - :param str schema_password: The password to be used to admin the database. - :param str state: I have no idea what this is for. TODO: Document this better - """ - conversation = self.conversation(scope=service) - conversation.set_remote( - host=host, - port=port, - database=database, - user=user, - password=password, - schema_user=schema_user, - schema_password=schema_password, - state=state, - ) - conversation.set_local('database', database) - conversation.remove_state('{relation_name}.database.requested') - - @not_unless('{provides:pgsql}.roles.requested') - def ack_roles(self, service, roles): - """ - Acknowledge that a set of roles have been given to a service's user. - - :param str service: The service which requested the roles, as - returned by :meth:`~provides.PostgreSQL.requested_roles`. - """ - conversation = self.conversation(scope=service) - conversation.set_local('roles', roles) - conversation.remove_state('{relation_name}.roles.requested') - - def requested_roles(self, service=None): - """ - Return the roles requested by all or a single given service. - - :param str service: The name of a service requesting roles, as - provided by either :meth:`requested_roles` (with no args) or - :meth:`requested_databases`. - :returns: If no service name is given, then a list of ``(service, roles)`` - tuples are returned, mapping service names to their requested - roles. If a service name is given, a list of the roles requested - for that service is returned. - - Example usage:: - - for service, roles in pgsql.requested_roles(): - set_roles(username_from_service(service), roles) - pgsql.ack_roles(service, roles) - """ - _roles = lambda conv: filter(None, conv.get_remote('roles', '').split(',')) - if service is not None: - return _roles(self.conversation(scope=service)) - else: - results = [] - for conversation in self.conversations(): - service = conversation.scope - results.append((service, _roles(conversation))) - return results - - def previous_roles(self, service): - """ - Return the roles previously requested, if different from the currently - requested roles. - """ - return self.conversation(scope=service).get_local('roles') - - def requested_databases(self): - """ - Return a list of tuples mapping a service name to the database name - requested by that service. If a given service has not requested a - specific database name, an empty string is returned, indicating that - the database name should be generated. - - Example usage:: - - for service, database in pgsql.requested_databases(): - database = database or generate_dbname(service) - pgsql.provide_database(**create_database(database)) - """ - for conversation in self.conversations(): - service = conversation.scope - database = self.requested_database(service) - yield service, database - - def requested_database(self, service): - """ - Return the database name requested by the given service. If the given - service has not requested a specific database name, an empty string is - returned, indicating that the database name should be generated. - """ - return self.conversation(scope=service).get_remote('database', '') - - def previous_database(self, service): - """ - Return the roles previously requested, if different from the currently - requested roles. - """ - return self.conversation(scope=service).get_local('database') diff --git a/requires.py b/requires.py index b81b538..46688a9 100644 --- a/requires.py +++ b/requires.py @@ -1,70 +1,306 @@ -#!/usr/bin/python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Copyright 2016 Canonical Ltd. # -# http://www.apache.org/licenses/LICENSE-2.0 +# This file is part of the PostgreSQL Client Interface for Juju charms.reactive # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from collections import namedtuple +import ipaddress +import itertools +import urllib.parse + +from charmhelpers import context +from charmhelpers.core import hookenv +from charms.reactive import hook, scopes, RelationBase + + +# This data structure cannot be in an external library, as interfaces +# have no way to declare dependencies. It also must be defined in this +# file, as reactive framework imports are broken per +# https://github.com/juju-solutions/charms.reactive/pull/51 +# +class ConnectionString(str): + """A libpq connection string. + + >>> c = ConnectionString(host='1.2.3.4', dbname='mydb', + ... port=5432, user='anon', password='secret') + ... + >>> c + 'host=1.2.3.4 dbname=mydb port=5432 user=anon password=secret + + Components may be accessed as attributes. + + >>> c.dbname + 'mydb' + >>> c.host + '1.2.3.4' + >>> c.port + '5432' + + The standard URI format is also accessible: + + >>> c.uri + 'postgresql://anon:secret@1.2.3.4:5432/mydb' + + """ + def __new__(self, **kw): + def quote(x): + return str(x).replace("\\", "\\\\").replace("'", "\\'") + c = " ".join("{}={}".format(k, quote(v)) for k, v in kw.items()) + c = str.__new__(self, c) + + for k, v in kw.items(): + setattr(c, k, v) + + self._keys = set(kw.keys()) -from charms.reactive import RelationBase -from charms.reactive import hook -from charms.reactive import scopes + d = {k: urllib.parse.quote(v, safe='') for k, v in kw.items()} + try: + hostaddr = ipaddress.ip_address(kw.get('hostaddr') or + kw.get('host')) + if isinstance(hostaddr, ipaddress.IPv6Address): + d['host'] = '[{}]'.format(hostaddr) + else: + d['host'] = str(hostaddr) + except ValueError: + pass + fmt = 'postgresql://{user}:{password}@{host}:{port}/{dbname}' + self.uri = fmt.format(**d) + + return c + + host = None + dbname = None + port = None + user = None + password = None + uri = None + + def keys(self): + return iter(self._keys) + + def items(self): + return {k: self[k] for k in self.keys()}.items() + + def values(self): + return iter(self[k] for k in self.keys()) + + def __getitem__(self, key): + if isinstance(key, int): + return super(ConnectionString, self).__getitem__(key) + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) + + +ConnectionStrings = namedtuple('ConnectionStrings', + ['relid', 'master', 'standbys']) +ConnectionStrings.__doc__ = ( + """Collection of :class:`ConnectionString` for a relation.""") +ConnectionStrings.relid.__doc__ = 'Relation id' +ConnectionStrings.master.__doc__ = 'master database :class:`ConnectionString`' +ConnectionStrings.standbys.__doc__ = ( + 'set of :class:`ConnectionString` to hot standby databases') class PostgreSQLClient(RelationBase): - # We only expect a single pgsql server to be related. Additionally, if - # there are multiple units, it would be for replication purposes only, - # so we would expect a leader to provide our connection info, or at least - # for all pgsql units to agree on the connection info. Thus, we use a - # global conversation scope in which all services and units share the - # same conversation. - scope = scopes.GLOBAL - - # These remote data fields will be automatically mapped to accessors - # with a basic documentation string provided. - auto_accessors = ['host', 'port', 'database', 'user', 'password', - 'schema_user', 'schema_password'] - - @hook('{requires:pgsql}-relation-{joined,changed}') - def changed(self): + """ + PostgreSQL client interface. + + A client may be related to one or more PostgreSQL services. + + In most cases, a charm will only use a single PostgreSQL + service being related for each relation defined in metadata.yaml + (so one per relation name). To access the connection strings, use + the master and standbys attributes:: + + @when('productdb.master.available') + def setup_database(pgsql): + conn_str = pgsql.master # A ConnectionString. + update_db_conf(conn_str) + + @when('productdb.standbys.available') + def setup_cache_databases(pgsql): + set_cache_db_list(pgsql.standbys) # set of ConnectionString. + + In somecases, a relation name may be related to several PostgreSQL + services. You can also access the ConnectionStrings for a particular + service by relation id or by iterating over all of them:: + + @when('db.master.available') + def set_dbs(pgsql): + update_monitored_dbs(cs.master + for cs in pgsql # ConnectionStrings. + if cs.master) + """ + scope = scopes.SERVICE + + @hook('{requires:pgsql}-relation-joined') + def joined(self): + # There is at least one named relation self.set_state('{relation_name}.connected') - if self.connection_string(): - self.set_state('{relation_name}.database.available') + hookenv.log('Joined {} relation'.format(hookenv.relation_id())) - def request_roles(self, *roles): - """ - Tell the PostgreSQL server to provide our user with a certain set of roles. + @hook('{requires:pgsql}-relation-{joined,changed,departed}') + def changed(self): + relid = hookenv.relation_id() + cs = self[relid] - :param list roles: One or more role names to give to this service's user. - """ - self.set_remote('roles', ','.join(roles)) + # There is a master in this relation. + self.toggle_state('{relation_name}.master.available', + cs.master) - def change_database_name(self, dbname): - """ - Tell the PostgreSQL server to provide us with a database with a specific name. + # There is at least one standby in this relation. + self.toggle_state('{relation_name}.standbys.available', + cs.standbys) - :param str dbname: New name for the database to use. - """ - self.set_remote('database', dbname) + # There is at least one database in this relation. + self.toggle_state('{relation_name}.database.available', + cs.master or cs.standbys) + + # Ideally, we could turn logging off using a layer option + # but that is not available for interfaces. + if cs.master and cs.standbys: + hookenv.log('Relation {} has master and standby ' + 'databases available'.format(relid)) + elif cs.master: + hookenv.log('Relation {} has a master database available, ' + 'but no standbys'.format(relid)) + elif cs.standbys: + hookenv.log('Relation {} only has standby databases ' + 'available'.format(relid)) + else: + hookenv.log('Relation {} has no databases available'.format(relid)) + + @hook('{requires:pgsql}-relation-departed') + def departed(self): + if not any(u for u in hookenv.related_units() or [] + if u != hookenv.remote_unit()): + self.remove_state('{relation_name}.connected') + self.conversation().depart() + hookenv.log('Departed {} relation'.format(hookenv.relation_id())) + + def set_database(self, dbname, relid=None): + """Set the database that the named relations connect to. + + The PostgreSQL service will create the database if necessary. It + will never remove it. + + :param dbname: The database name. If unspecified, the local service + name is used. + + :param relid: relation id to send the database name setting to. + If unset, the setting is broadcast to all relations + sharing the relation name. - def connection_string(self): """ - Get the connection string, if available, or None. + for c in self.conversations(): + if relid is None or c.namespace == relid: + c.set_remote('database', dbname) + + def __getitem__(self, relid): + """:returns: :class:`ConnectionStrings` for the relation id.""" + relations = context.Relations() + relname = self.relation_name + assert relid.startswith('{}:'.format(relname)), ( + 'relid {} not handled by {} instance'.format(relid, relname)) + + relation = relations[relname][relid] + + master_reldatas = [reldata + for reldata in relation.values() + if reldata.get('state') in ('master', 'standalone')] + if len(master_reldatas) == 1: + master_reldata = master_reldatas[0] # One, and only one. + else: + # None ready, or multiple due to failover in progress. + master_reldata = None + + master = self._cs(master_reldata) + + standbys = set(filter(None, + (self._cs(reldata) + for reldata in relation.values() + if reldata.get('state') == 'hot standby'))) + + return ConnectionStrings(relid, master, standbys) + + def __iter__(self): + """:returns: Iterator of :class:`ConnectionStrings` for this named + relation, one per relation id. """ - data = { - 'host': self.host(), - 'port': self.port(), - 'database': self.database(), - 'user': self.user(), - 'password': self.password(), - } - if all(data.values()): - return str.format( - 'host={host} port={port} dbname={database} user={user} password={password}', - **data) - return None + return iter(self[relid] + for relid in context.Relations()[self.relation_name]) + + @property + def master(self): + ''':class:`ConnectionString` to the master, or None. + + If multiple PostgreSQL services are related using this relation + name then the first master found is returned. + ''' + for cs in self: + if cs.master: + return cs.master + + @property + def standbys(self): + '''Set of class:`ConnectionString` to the read-only hot standbys. + + If multiple PostgreSQL services are related using this relation + name then all standbys found are returned. + ''' + return set(itertools.chain(*(cs.standbys for cs in self))) + + def connection_string(self, unit=None): + ''':class:`ConnectionString` to the remote unit, or None. + + unit defaults to the active remote unit. + + You should normally use the master or standbys attributes rather + than this method. + + If the unit is related multiple times using the same relation + name, the first one found is returned. + ''' + if unit is None: + unit = hookenv.remote_unit() + + relations = context.Relations() + for relation in relations[self.relation_name].values(): + if unit in relation: + conn_str = self._cs(relation[unit]) + if conn_str: + return conn_str + raise LookupError(unit) + + def _cs(self, reldata): + if not reldata: + return None + d = dict(host=reldata.get('host'), + port=reldata.get('port'), + dbname=reldata.get('database'), + user=reldata.get('user'), + password=reldata.get('password')) + if not all(d.values()): + return None + local_unit = hookenv.local_unit() + if local_unit not in reldata.get('allowed-units', '').split(): + # Not yet authorized + return None + locdata = context.Relations()[reldata.relname][reldata.relid].local + if 'database' in locdata and locdata['database'] != d['dbname']: + # Requested database does not yet match + return None + return ConnectionString(**d) diff --git a/test_requirements.txt b/test_requirements.txt index d033b51..d8064e3 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -4,7 +4,7 @@ coverage>=3.6 mock>=1.0.1 nose>=1.3.1 flake8 --e bzr+http://bazaar.launchpad.net/~johnsca/charm-helpers/reactive/#egg=charmhelpers -# -# Specify precise versions of runtime dependencies where possible. -PyYAML==3.10 # precise +PyYAML +sphinx +charms.reactive +charmhelpers