From bf15db7f52d0a36355ac938b94e26be47a85097e Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Wed, 9 Nov 2016 01:39:22 +0500 Subject: [PATCH 01/36] Get certs through letsencrypt --- .gitignore | 1 + Dockerfile | 6 +- requirements.txt | 3 + src/app.py | 11 +- src/setup.py | 3 + src/vergilius/__init__.py | 25 ++- .../components/acme_certificate_provider.py | 196 ++++++++++++++++++ .../components/dummy_certificate_provider.py | 6 - src/vergilius/loop/nginx_reloader.py | 9 +- src/vergilius/loop/service_watcher.py | 6 +- src/vergilius/models/certificate.py | 73 +++---- src/vergilius/models/identity.py | 4 +- src/vergilius/models/service.py | 68 +++--- src/vergilius/session.py | 52 +++++ src/vergilius/templates/service_stub.html | 15 ++ 15 files changed, 389 insertions(+), 89 deletions(-) create mode 100644 src/vergilius/components/acme_certificate_provider.py create mode 100644 src/vergilius/session.py create mode 100644 src/vergilius/templates/service_stub.html diff --git a/.gitignore b/.gitignore index d6db400..4afa3a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +.*.swp /*.egg-info /.tox diff --git a/Dockerfile b/Dockerfile index 67b2f0d..5803c3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ MAINTAINER Vasiliy Ostanin RUN add-apt-repository ppa:nginx/development RUN apt-get update #apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \ -RUN apt-get install -y ca-certificates nginx git-core python build-essential autoconf libtool \ - python-dev libffi-dev libssl-dev python-pip dialog nano +RUN apt-get install -y ca-certificates nginx git-core python3 build-essential autoconf libtool \ + python3-dev libffi-dev libssl-dev python3-pip dialog nano ENV TERM screen ADD init.d/01_env.sh /etc/init.d/ @@ -19,7 +19,7 @@ RUN rm /etc/nginx/sites-enabled/* && mkdir -p /etc/nginx/sites-enabled/certs && mkdir -p /data/dummy_ca/domains/ ADD src /opt/vergilius -RUN cd /opt/vergilius/ && python setup.py install +RUN cd /opt/vergilius/ && python3 setup.py install WORKDIR /opt/vergilius/ EXPOSE 80 443 diff --git a/requirements.txt b/requirements.txt index 53fbe21..7156bfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,6 @@ python-consul==0.4.7 tornado==4.3 funcsigs==1.0.0 mock==1.3.0 +acme==0.9.3 +cryptography>=0.8 +PyOpenSSL>=0.13 diff --git a/src/app.py b/src/app.py index 8e2cd10..24719d9 100755 --- a/src/app.py +++ b/src/app.py @@ -1,12 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python3 import logging import signal import time import tornado -import vergilius -from vergilius import logger +from vergilius import logger, Vergilius from vergilius.loop.nginx_reloader import NginxReloader from vergilius.loop.service_watcher import ServiceWatcher @@ -41,15 +40,15 @@ def sig_handler(sig, frame): def main(): + io_loop = tornado.ioloop.IOLoop.current(); signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) - vergilius.Vergilius.init() + v = Vergilius() consul_handler = ServiceWatcher() nginx_reloader = NginxReloader() - - tornado.ioloop.IOLoop.current().start() + io_loop.start() if __name__ == '__main__': diff --git a/src/setup.py b/src/setup.py index 4bb25c8..3d125dc 100644 --- a/src/setup.py +++ b/src/setup.py @@ -7,6 +7,9 @@ 'setuptools>=1.0', 'zope.component', 'zope.interface', + 'acme==0.9.3', + 'cryptography>=0.8', + 'PyOpenSSL>=0.13', ] setup( diff --git a/src/vergilius/__init__.py b/src/vergilius/__init__.py index 7f9268f..3fb3cef 100644 --- a/src/vergilius/__init__.py +++ b/src/vergilius/__init__.py @@ -5,13 +5,14 @@ from consul import tornado as consul_from_tornado from tornado import template -import config -from components.dummy_certificate_provider import DummyCertificateProvider +import vergilius.config +from .components.dummy_certificate_provider import DummyCertificateProvider +from .components.acme_certificate_provider import AcmeCertificateProvider from vergilius.models.identity import Identity +from vergilius.session import ConsulSession logger = logging.getLogger(__name__) template_loader = template.Loader(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')) -certificate_provider = DummyCertificateProvider() consul = Consul(host=config.CONSUL_HOST) consul_tornado = consul_from_tornado.Consul(host=config.CONSUL_HOST) @@ -20,6 +21,18 @@ class Vergilius(object): identity = None - @classmethod - def init(cls): - cls.identity = Identity() + __instance = None + + def __new__(cls): + if Vergilius.__instance is None: + Vergilius.__instance = object.__new__(cls) + Vergilius.__instance.init() + return Vergilius.__instance + + def instance(): + return Vergilius.__instance + + def init(self): + self.session = ConsulSession() + self.identity = Identity() + self.certificate_provider = AcmeCertificateProvider(session=self.session) diff --git a/src/vergilius/components/acme_certificate_provider.py b/src/vergilius/components/acme_certificate_provider.py new file mode 100644 index 0000000..ddd40ea --- /dev/null +++ b/src/vergilius/components/acme_certificate_provider.py @@ -0,0 +1,196 @@ +import os, sys, base64, time, logging +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime + +import tornado.web, tornado.gen +from tornado.httpclient import HTTPClient +from vergilius import config +from consul.tornado import Consul as TornadoConsul +from consul import Consul + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL + +from acme import client, messages, jose +import time + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +thread_pool = ThreadPoolExecutor(4) + +DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' + +class AcmeCertificateProvider(object): + tc = TornadoConsul(host=config.CONSUL_HOST) + cc = Consul(host=config.CONSUL_HOST) + private_key = None + + def __init__(self, *, session): + self.session = session + self.app = self.make_app() + self.app.listen(8888) + self.fetch_key() + self.init_acme() + + def make_app(self): + return tornado.web.Application([ + (r"/.well-known/acme-challenge/(.+)", AcmeChallengeHandler), + ]) + + def fetch_key(self): + index, key_data = self.cc.kv.get('vergilius/acme/private_key') + if (key_data): + self.private_key = serialization.load_pem_private_key(key_data['Value'],password=None,backend=default_backend()) + else: + self.private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + key_data = self.private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + self.cc.kv.put('vergilius/acme/private_key', key_data) + self.acme_key = jose.JWKRSA(key=self.private_key) + + def init_acme(self): + self._acme = client.Client(DIRECTORY_URL, self.acme_key) + try: + regr = self._acme.register() + acme.agree_to_tos(self._regr) + except: + pass + + def get_for_domain(self, domain): + + def _b64(b): + return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") + + # get token + def _parse_token(authzr): + token = None + for c in authzr.body.challenges: + json = c.chall.to_partial_json() + if json['type'] == 'http-01': + return json['token'] + + def _store_token(token): + # put token to consul KV + thumbprint = _b64(self.acme_key.thumbprint()) + keyauth = '{0}.{1}'.format(token,thumbprint) + self.cc.kv.put('vergilius/acme/challenge/%s' % token, keyauth) + + # request challenges for domain + authzr = self._acme.request_domain_challenges(domain) + token = _parse_token(authzr) + _store_token(token) + + challenge = [ x for x in authzr.body.challenges if x.typ == 'http-01' ][0] + response, validation = challenge.response_and_validation(self.acme_key) + print('chall uri', challenge.uri) + + result = self._acme.answer_challenge(challenge, response) + print('answer result is ', result) + + waitUntil = time.time() + 30 + while time.time() < waitUntil: + logger.debug('polling...') + authzr, authzr_response = self._acme.poll(authzr) + if authzr.body.status not in ( + messages.STATUS_VALID, messages.STATUS_INVALID): + time.sleep(2) + else: + break + logger.debug(authzr) + return authzr + + def get_csr(self, domains): + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + firstDomain = domains[0] + csr = x509.CertificateSigningRequestBuilder().subject_name( + x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, 'RU'), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'Yekaterinburg'), + x509.NameAttribute(NameOID.LOCALITY_NAME, 'Yekaterinburg'), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'devopsftw'), + x509.NameAttribute(NameOID.COMMON_NAME, firstDomain), + ]) + ).add_extension( + x509.SubjectAlternativeName([ + x509.DNSName(domain) for domain in domains + ]), + critical = False, + ).sign(private_key, hashes.SHA256(), default_backend()) + + csr_openssl = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, + csr.public_bytes(serialization.Encoding.PEM) + ) + return (private_key, csr_openssl) + + def get_authzrs(self, domains): + authzrs = [ self.get_for_domain(domain) for domain in domains ] + return authzrs + + def request_certificate(self, csr, domains, authzrs): + try: + response = self._acme.request_issuance(jose.util.ComparableX509(csr), authzrs) + cert_data = HTTPClient().fetch(response.uri).body + cert = x509.load_der_x509_certificate(cert_data, default_backend()) + return cert + except messages.Error as error: + print ("This script is doomed to fail as no authorization " + "challenges are ever solved. Error from server: {0}".format(error)) + return None + + def query_letsencrypt(self, domains): + authzrs = self.get_authzrs(domains) + domain_key, csr = self.get_csr(domains) + cert = self.request_certificate(csr, domains, authzrs) + return (domain_key, cert) + + @tornado.gen.coroutine + def get_certificate(self, id, domains): + logger.debug('get cert for domains %s' % domains) + sid = yield self.session.get_sid() + logger.debug('sid is %s' % sid) + + domain_key, cert = yield thread_pool.submit(self.query_letsencrypt, domains) + + key_str = domain_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + cert_str = cert.public_bytes(serialization.Encoding.PEM) + expires = int(cert.not_valid_after.timestamp()) + result = { + 'private_key': key_str, + 'public_key': cert_str, + 'expires' : expires + + } + logger.debug('cert result is', result) + return result + +class AcmeChallengeHandler(tornado.web.RequestHandler): + tc = TornadoConsul(host=config.CONSUL_HOST) + + @tornado.gen.coroutine + def get(self, challenge): + logger.debug('challenge request: %s' % challenge) + index, data = yield self.tc.kv.get('vergilius/acme/challenge/%s' % challenge) + if data: + self.write(data['Value']) + else: + raise tornado.web.HTTPError(404) diff --git a/src/vergilius/components/dummy_certificate_provider.py b/src/vergilius/components/dummy_certificate_provider.py index 5ed252b..90fbf28 100644 --- a/src/vergilius/components/dummy_certificate_provider.py +++ b/src/vergilius/components/dummy_certificate_provider.py @@ -3,7 +3,6 @@ import os import subprocess import time -import zope.interface import vergilius from vergilius.components.certificate_provider import ICertificateProvider @@ -29,11 +28,6 @@ def check_paths(): class DummyCertificateProvider(object): - """ - Issues self signed certificates. - """ - zope.interface.implements(ICertificateProvider) - @classmethod def dfile(cls, id, ext): return os.path.join(DATA_PATH, 'domains', '%s.%s' % (id, ext)) diff --git a/src/vergilius/loop/nginx_reloader.py b/src/vergilius/loop/nginx_reloader.py index 35a9c10..29abe4a 100644 --- a/src/vergilius/loop/nginx_reloader.py +++ b/src/vergilius/loop/nginx_reloader.py @@ -1,15 +1,17 @@ import subprocess from consul import tornado +from tornado.ioloop import IOLoop from tornado.locks import Event import vergilius +from vergilius import logger class NginxReloader(object): nginx_update_event = Event() def __init__(self): - self.nginx_reload() + IOLoop.instance().spawn_callback(self.nginx_reload) @classmethod @tornado.gen.coroutine @@ -18,7 +20,10 @@ def nginx_reload(cls): yield cls.nginx_update_event.wait() cls.nginx_update_event.clear() vergilius.logger.info('[nginx]: reload') - subprocess.check_call([vergilius.config.NGINX_BINARY, '-s', 'reload']) + try: + subprocess.check_call([vergilius.config.NGINX_BINARY, '-s', 'reload'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logger.error('failed to reload nginx') @classmethod def queue_reload(cls): diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop/service_watcher.py index 7bf4c9f..958fa75 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop/service_watcher.py @@ -1,16 +1,16 @@ import vergilius +from tornado.ioloop import IOLoop from consul import tornado, base from vergilius.models.service import Service - class ServiceWatcher(object): def __init__(self): self.services = {} self.data = {} self.modified = False - self.watch_services() + IOLoop.instance().spawn_callback(self.watch_services) @tornado.gen.coroutine def watch_services(self): @@ -32,6 +32,6 @@ def check_services(self, data): # cleanup stale services for service_name in self.services.keys(): - if service_name not in services_to_publish.iterkeys(): + if service_name not in services_to_publish.keys(): vergilius.logger.info('[service watcher]: removing stale service: %s' % service_name) del self.services[service_name] diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py index 9e45888..c320c2a 100644 --- a/src/vergilius/models/certificate.py +++ b/src/vergilius/models/certificate.py @@ -1,13 +1,17 @@ import os import time from tornado import ioloop +from tornado.locks import Event +import tornado.gen -from consul import tornado, base -from vergilius import consul, logger, certificate_provider, config - +import consul +from consul.tornado import Consul as TornadoConsul +from vergilius import Vergilius, logger, config class Certificate(object): - tc = tornado.Consul(host=config.CONSUL_HOST) + tc = TornadoConsul(host=config.CONSUL_HOST) + cc = consul.Consul(host=config.CONSUL_HOST) + ready_event = Event() def __init__(self, service, domains): """ @@ -30,24 +34,24 @@ def __init__(self, service, domains): os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) ioloop.IOLoop.instance().add_callback(self.unlock) - self.fetch() - self.watch() + ioloop.IOLoop.instance().spawn_callback(self.watch) - def fetch(self): - index, data = consul.kv.get('vergilius/certificates/%s/' % self.service.id, recurse=True) - self.load_keys_from_consul(data) + @tornado.gen.coroutine + def fetch(self, index): + index, data = yield self.tc.kv.get('vergilius/certificates/%s/' % self.service.id, index=index, recurse=True) + return (index, data) @tornado.gen.coroutine def watch(self): index = None while True and self.active: try: - index, data = \ - yield self.tc.kv.get('vergilius/certificates/%s/' % self.service.id, index=index, recurse=True) - self.load_keys_from_consul(data) - except base.Timeout: + index, data = yield self.fetch(index) + yield self.load_keys_from_consul(data) + except consul.base.Timeout: pass + @tornado.gen.coroutine def load_keys_from_consul(self, data=None): if data: for item in data: @@ -58,23 +62,24 @@ def load_keys_from_consul(self, data=None): if not self.validate(): logger.warn('[certificate][%s]: cant validate existing keys' % self.service.id) self.discard_certificate() - if not self.request_certificate(): + if not (yield self.request_certificate()): return False else: logger.debug('[certificate][%s]: using existing keys' % self.service.id) else: - if not self.request_certificate(): + if not (yield self.request_certificate()): return False self.write_certificate_files() + self.ready_event.set() return True def write_certificate_files(self): - key_file = open(self.get_key_path(), 'w+') + key_file = open(self.get_key_path(), 'wb') key_file.write(self.private_key) key_file.close() - pem_file = open(self.get_cert_path(), 'w+') + pem_file = open(self.get_cert_path(), 'wb') pem_file.write(self.public_key) pem_file.close() @@ -90,45 +95,43 @@ def get_key_path(self): def get_cert_path(self): return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.pem') - def lock(self): + def acquire_lock(self): """ Create a lock in consul to prevent certificate request race condition """ - self.lock_session_id = consul.session.create(behavior='delete') - return consul.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) + self.lock_session_id = self.cc.session.create(behavior='delete',ttl=10) + return self.cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) def unlock(self): if not self.lock_session_id: return - consul.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) - consul.session.destroy(self.lock_session_id) + self.cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) + self.cc.session.destroy(self.lock_session_id) self.lock_session_id = None + @tornado.gen.coroutine def request_certificate(self): logger.debug('[certificate][%s] Requesting new keys for %s ' % (self.service.name, self.domains)) - if not self.lock(): + if not self.acquire_lock(): logger.debug('[certificate][%s] failed to acquire lock for keys generation' % self.service.name) return False try: - data = certificate_provider.get_certificate(self.service.id, self.domains) + data = yield Vergilius.instance().certificate_provider.get_certificate(self.service.id, self.domains) - with open(data['private_key'], 'r') as f: - self.private_key = f.read() - f.close() - consul.kv.put('vergilius/certificates/%s/private_key' % self.service.id, self.private_key) + self.private_key = data['private_key'] + self.cc.kv.put('vergilius/certificates/%s/private_key' % self.service.id, self.private_key) - with open(data['public_key'], 'r') as f: - self.public_key = f.read() - f.close() - consul.kv.put('vergilius/certificates/%s/public_key' % self.service.id, self.public_key) + self.public_key = data['public_key'] + self.cc.kv.put('vergilius/certificates/%s/public_key' % self.service.id, self.public_key) self.expires = data['expires'] self.key_domains = self.serialize_domains() - consul.kv.put('vergilius/certificates/%s/expires' % self.service.id, str(self.expires)) - consul.kv.put('vergilius/certificates/%s/key_domains' % self.service.id, self.serialize_domains()) + logger.debug('write domain %s' % self.key_domains) + self.cc.kv.put('vergilius/certificates/%s/expires' % self.service.id, str(self.expires)) + self.cc.kv.put('vergilius/certificates/%s/key_domains' % self.service.id, self.serialize_domains()) logger.info('[certificate][%s]: got new keys for %s ' % (self.service.name, self.domains)) self.write_certificate_files() except Exception as e: @@ -138,7 +141,7 @@ def request_certificate(self): self.unlock() def serialize_domains(self): - return '|'.join(sorted(self.domains)) + return '|'.join(sorted(self.domains)).encode() def discard_certificate(self): pass diff --git a/src/vergilius/models/identity.py b/src/vergilius/models/identity.py index 7211603..0137bfc 100644 --- a/src/vergilius/models/identity.py +++ b/src/vergilius/models/identity.py @@ -76,10 +76,10 @@ def write_files(self): if not self.get_private_key(): self.generate_identity() - private_key_file = open(self.get_private_key_path(), 'w+') + private_key_file = open(self.get_private_key_path(), 'wb') private_key_file.write(self.get_private_key()) private_key_file.close() - certificate_file = open(self.get_certificate_path(), 'w+') + certificate_file = open(self.get_certificate_path(), 'wb') certificate_file.write(self.get_certificate()) certificate_file.close() diff --git a/src/vergilius/models/service.py b/src/vergilius/models/service.py index 5cd2dec..5618dac 100644 --- a/src/vergilius/models/service.py +++ b/src/vergilius/models/service.py @@ -4,6 +4,8 @@ import tempfile import unicodedata +from tornado.ioloop import IOLoop + from consul import tornado, base from vergilius import config, consul_tornado, consul, logger, template_loader from vergilius.loop.nginx_reloader import NginxReloader @@ -11,6 +13,8 @@ class Service(object): + active = False + def __init__(self, name): """ :type name: unicode - service name got from consul @@ -21,8 +25,8 @@ def __init__(self, name): self.allow_crossdomain = False self.nodes = {} self.domains = { - u'http': set(), - u'http2': set() + 'http': set(), + 'http2': set() } self.active = True @@ -31,12 +35,8 @@ def __init__(self, name): if not os.path.exists(config.NGINX_CONFIG_PATH): os.mkdir(config.NGINX_CONFIG_PATH) - self.fetch() - self.watch() - - def fetch(self): - index, data = consul.health.service(self.name, passing=True) - self.parse_data(data) + # spawn service watcher + IOLoop.instance().spawn_callback(self.watch) @tornado.gen.coroutine def watch(self): @@ -44,16 +44,19 @@ def watch(self): while True and self.active: try: index, data = yield consul_tornado.health.service(self.name, index, wait=None, passing=True) - self.parse_data(data) + yield self.parse_data(data) + # okay, got data, now reload + yield self.update_config() except base.Timeout: pass + @tornado.gen.coroutine def parse_data(self, data): """ :type data: set[] """ - for protocol in self.domains.iterkeys(): + for protocol in self.domains.keys(): self.domains[protocol].clear() allow_crossdomain = False @@ -85,22 +88,38 @@ def parse_data(self, data): self.allow_crossdomain = allow_crossdomain - self.flush_nginx_config() + @tornado.gen.coroutine + def update_config(self): + # if we have http2 domain, create stub nginx config for ACME + if self.domains[u'http2']: + + # if we dont have certificate yet, create stub config and wait + if not self.certificate: + logger.debug('[service][%s] flush stub config' % self.id) + self.flush_nginx_config(self.get_stub_config()) + self.certificate = Certificate(service=self, domains=self.domains[u'http2']) + logger.debug('[service][%s] wait for cert' % self.id) + yield self.certificate.ready_event.wait() + logger.debug('[service][%s] load real https config' % self.id) + self.flush_nginx_config(self.get_nginx_config()) + else: + logger.debug('[service][%s] flush real config' % self.id) + self.flush_nginx_config(self.get_nginx_config()) def get_nginx_config(self): """ Generate nginx config from service attributes """ - if self.domains[u'http2']: - self.check_certificate() return template_loader.load('service.html').generate(service=self, config=config) - def flush_nginx_config(self): - if not self.validate(): + def get_stub_config(self): + return template_loader.load('service_stub.html').generate(service=self, config=config) + + def flush_nginx_config(self, nginx_config): + if not self.validate(nginx_config): logger.error('[service][%s]: failed to validate nginx config!' % self.id) return False - nginx_config = self.get_nginx_config() deployed_nginx_config = None try: @@ -109,7 +128,7 @@ def flush_nginx_config(self): pass if deployed_nginx_config != nginx_config: - config_file = open(self.get_nginx_config_path(), 'w+') + config_file = open(self.get_nginx_config_path(), 'wb') config_file.write(nginx_config) config_file.close() logger.info('[service][%s]: got new nginx config %s' % (self.name, self.get_nginx_config_path())) @@ -124,13 +143,13 @@ def read_nginx_config_file(self): config_file.close() return config_content - def validate(self): + def validate(self, config_str): """ Deploy temporary service & nginx config and validate it with nginx :return: bool """ service_config_file = tempfile.NamedTemporaryFile(delete=False) - service_config_file.write(self.get_nginx_config()) + service_config_file.write(config_str) service_config_file.close() nginx_config_file = tempfile.NamedTemporaryFile(delete=False) @@ -141,7 +160,7 @@ def validate(self): nginx_config_file.close() try: - return_code = subprocess.check_call([config.NGINX_BINARY, '-t', '-c', nginx_config_file.name]) + return_code = subprocess.check_call([config.NGINX_BINARY, '-t', '-c', nginx_config_file.name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError: return_code = 1 finally: @@ -174,10 +193,7 @@ def slugify(cls, string): Normalizes string, converts to lowercase, removes non-alpha characters, and converts spaces to hyphens. """ - string = unicodedata.normalize('NFKD', unicode(string)).encode('ascii', 'ignore') - string = unicode(re.sub('[^\w\s-]', '', string).strip().lower()) + string = unicodedata.normalize('NFKD', string) + string = string.encode('ascii', 'ignore').decode() + string = str(re.sub('[^\w\s-]', '', str(string)).strip().lower()) return re.sub('[-\s]+', '-', string) - - def check_certificate(self): - if not self.certificate: - self.certificate = Certificate(service=self, domains=self.domains[u'http2']) diff --git a/src/vergilius/session.py b/src/vergilius/session.py new file mode 100644 index 0000000..90bf7b3 --- /dev/null +++ b/src/vergilius/session.py @@ -0,0 +1,52 @@ +import logging + +from vergilius import config + +import consul, tornado +from consul.tornado import Consul as TornadoConsul +from tornado.ioloop import IOLoop +from tornado.locks import Event + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +class ConsulSession(object): + tc = TornadoConsul(host=config.CONSUL_HOST) + cc = consul.Consul(host=config.CONSUL_HOST) + _sid = None + + def __init__(self): + self._waitSid = Event() + IOLoop.instance().spawn_callback(self.watch) + pass + + @tornado.gen.coroutine + def watch(self): + while True: + tick = tornado.gen.sleep(5) + yield self.ensure_session() + yield tick + + @tornado.gen.coroutine + def ensure_session(self): + if self._sid == None: + self._sid = yield self.create_session() + self._waitSid.set() + else: + try: + yield self.tc.session.renew(self._sid) + except consul.NotFound: + logger.error('session not found, trying to recreate') + self._sid = yield self.create_session() + return True + + @tornado.gen.coroutine + def create_session(self): + sid = yield self.tc.session.create('vergilius', ttl=10, behavior='delete', lock_delay=0) + logger.debug('session created: %s', sid) + return sid + + @tornado.gen.coroutine + def get_sid(self): + yield self._waitSid.wait() + return self._sid diff --git a/src/vergilius/templates/service_stub.html b/src/vergilius/templates/service_stub.html new file mode 100644 index 0000000..f77421d --- /dev/null +++ b/src/vergilius/templates/service_stub.html @@ -0,0 +1,15 @@ +{% whitespace all%} +{% if len(service.domains['http2']) %} +server { + server_name{% for domain in service.domains['http2'] %} {{ domain }} *.{{ domain }}{% end %}; + listen {{config.NGINX_HTTP_PORT}}; + + location / { + return 503; + } + + location /.well-known/ { + proxy_pass http://127.0.0.1:8888; + } +} +{% end %} From 2e36d86a19546444761168a50df59f569b1cc402 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 02:44:22 +0500 Subject: [PATCH 02/36] pin package requirements --- requirements.txt | 4 ++-- src/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7156bfe..2d779fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ tornado==4.3 funcsigs==1.0.0 mock==1.3.0 acme==0.9.3 -cryptography>=0.8 -PyOpenSSL>=0.13 +cryptography==1.6 +PyOpenSSL==16.2 diff --git a/src/setup.py b/src/setup.py index 3d125dc..c8e629d 100644 --- a/src/setup.py +++ b/src/setup.py @@ -8,8 +8,8 @@ 'zope.component', 'zope.interface', 'acme==0.9.3', - 'cryptography>=0.8', - 'PyOpenSSL>=0.13', + 'cryptography==1.6', + 'PyOpenSSL==16.2', ] setup( From 546941d83205a793297221a9523ea41d06526416 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 02:54:32 +0500 Subject: [PATCH 03/36] handle ConsulException --- src/vergilius/loop/service_watcher.py | 15 +++++++++------ src/vergilius/models/certificate.py | 10 +++++++--- src/vergilius/models/service.py | 9 +++++++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop/service_watcher.py index 958fa75..dd5520e 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop/service_watcher.py @@ -1,8 +1,8 @@ -import vergilius - +import tornado.gen from tornado.ioloop import IOLoop -from consul import tornado, base +from consul import base, ConsulException from vergilius.models.service import Service +from vergilius import consul_tornado, logger class ServiceWatcher(object): def __init__(self): @@ -17,8 +17,11 @@ def watch_services(self): index = None while True: try: - index, data = yield vergilius.consul_tornado.catalog.services(index, wait=None) + index, data = yield consul_tornado.catalog.services(index, wait=None) self.check_services(data) + except ConsulException as e: + logger.error('consul error: %s' % e) + yield tornado.gen.sleep(5) except base.Timeout: pass @@ -27,11 +30,11 @@ def check_services(self, data): services_to_publish = dict((k, v) for k, v in data.items() if any(x in v for x in [u'http', u'http2'])) for service_name in services_to_publish: if service_name not in self.services: - vergilius.logger.info('[service watcher]: new service: %s' % service_name) + logger.info('[service watcher]: new service: %s' % service_name) self.services[service_name] = Service(service_name) # cleanup stale services for service_name in self.services.keys(): if service_name not in services_to_publish.keys(): - vergilius.logger.info('[service watcher]: removing stale service: %s' % service_name) + logger.info('[service watcher]: removing stale service: %s' % service_name) del self.services[service_name] diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py index c320c2a..92d245c 100644 --- a/src/vergilius/models/certificate.py +++ b/src/vergilius/models/certificate.py @@ -1,10 +1,11 @@ import os import time -from tornado import ioloop +from tornado.ioloop import IOLoop from tornado.locks import Event import tornado.gen import consul +from consul import ConsulException from consul.tornado import Consul as TornadoConsul from vergilius import Vergilius, logger, config @@ -33,8 +34,8 @@ def __init__(self, service, domains): if not os.path.exists(os.path.join(config.NGINX_CONFIG_PATH, 'certs')): os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) - ioloop.IOLoop.instance().add_callback(self.unlock) - ioloop.IOLoop.instance().spawn_callback(self.watch) + IOLoop.instance().add_callback(self.unlock) + IOLoop.instance().spawn_callback(self.watch) @tornado.gen.coroutine def fetch(self, index): @@ -48,6 +49,9 @@ def watch(self): try: index, data = yield self.fetch(index) yield self.load_keys_from_consul(data) + except ConsulException as e: + logger.error('consul error: %s' % e) + yield tornado.gen.sleep(5) except consul.base.Timeout: pass diff --git a/src/vergilius/models/service.py b/src/vergilius/models/service.py index 5618dac..7ecb6a4 100644 --- a/src/vergilius/models/service.py +++ b/src/vergilius/models/service.py @@ -4,9 +4,11 @@ import tempfile import unicodedata +import tornado.gen from tornado.ioloop import IOLoop -from consul import tornado, base +import consul.base +from consul import ConsulException from vergilius import config, consul_tornado, consul, logger, template_loader from vergilius.loop.nginx_reloader import NginxReloader from vergilius.models.certificate import Certificate @@ -47,7 +49,10 @@ def watch(self): yield self.parse_data(data) # okay, got data, now reload yield self.update_config() - except base.Timeout: + except ConsulException as e: + logger.error('consul error: %s' % e) + yield tornado.gen.sleep(5) + except consul.base.Timeout: pass @tornado.gen.coroutine From a3df1c2e36c21d05c1462947cbc573ad0626e775 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 02:56:40 +0500 Subject: [PATCH 04/36] use python 3.5 in CI --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 2c63ac9..ea495fa 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: python: - version: 2.7 + version: 3.5 services: - docker environment: @@ -25,4 +25,4 @@ deployment: dockerhub: branch: master commands: - - 'curl -H "Content-Type: application/json" --data "{\"source_type\": \"Branch\", \"source_name\": \"master\"}" -X POST https://registry.hub.docker.com/u/devopsftw/vergilius/trigger/ea3f932c-49b9-47e8-af0c-ec1d8615cda4/' \ No newline at end of file + - 'curl -H "Content-Type: application/json" --data "{\"source_type\": \"Branch\", \"source_name\": \"master\"}" -X POST https://registry.hub.docker.com/u/devopsftw/vergilius/trigger/ea3f932c-49b9-47e8-af0c-ec1d8615cda4/' From a3c16244919392f2a659e65f0f9273442d11a420 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 02:59:23 +0500 Subject: [PATCH 05/36] py 3.5.2 --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index ea495fa..a3ea40b 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: python: - version: 3.5 + version: 3.5.2 services: - docker environment: From 0a0ccefd5ef6e0d7807cbaadbddee0e8a3747ac9 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 03:27:10 +0500 Subject: [PATCH 06/36] ns and import fixes --- src/vergilius/loop/service_watcher.py | 7 +++++-- src/vergilius/models/certificate.py | 8 ++++---- src/vergilius/models/service.py | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop/service_watcher.py index dd5520e..3e73585 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop/service_watcher.py @@ -1,6 +1,9 @@ import tornado.gen from tornado.ioloop import IOLoop -from consul import base, ConsulException + +from consul import ConsulException +from consul.base import Timeout as ConsulTimeout + from vergilius.models.service import Service from vergilius import consul_tornado, logger @@ -22,7 +25,7 @@ def watch_services(self): except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except base.Timeout: + except ConsulTimeout: pass def check_services(self, data): diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py index 92d245c..fc5b7a1 100644 --- a/src/vergilius/models/certificate.py +++ b/src/vergilius/models/certificate.py @@ -4,14 +4,14 @@ from tornado.locks import Event import tornado.gen -import consul -from consul import ConsulException +from consul import Consul, ConsulException +from consul.base import Timeout as ConsulTimeout from consul.tornado import Consul as TornadoConsul from vergilius import Vergilius, logger, config class Certificate(object): tc = TornadoConsul(host=config.CONSUL_HOST) - cc = consul.Consul(host=config.CONSUL_HOST) + cc = Consul(host=config.CONSUL_HOST) ready_event = Event() def __init__(self, service, domains): @@ -52,7 +52,7 @@ def watch(self): except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except consul.base.Timeout: + except ConsulTimeout: pass @tornado.gen.coroutine diff --git a/src/vergilius/models/service.py b/src/vergilius/models/service.py index 7ecb6a4..04fa6df 100644 --- a/src/vergilius/models/service.py +++ b/src/vergilius/models/service.py @@ -7,8 +7,8 @@ import tornado.gen from tornado.ioloop import IOLoop -import consul.base from consul import ConsulException +from consul.base import Timeout as ConsulTimeout from vergilius import config, consul_tornado, consul, logger, template_loader from vergilius.loop.nginx_reloader import NginxReloader from vergilius.models.certificate import Certificate @@ -52,7 +52,7 @@ def watch(self): except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except consul.base.Timeout: + except ConsulTimeout: pass @tornado.gen.coroutine From 47a33b1caa80d0eebf12d0edb16d3f89974cfe96 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 24 Nov 2016 03:32:36 +0500 Subject: [PATCH 07/36] rm tests/init --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 From cd1fab557fd45bf48af10051339e89827845e0b7 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 23 Dec 2016 17:46:30 +0500 Subject: [PATCH 08/36] update structure for dockerfile build --- Dockerfile | 14 +++++++------- {consul => docker/consul}/vergilius-443.json | 0 {consul => docker/consul}/vergilius-80.json | 0 {consul => docker/consul}/vergilius-admin.json | 0 {init.d => docker/init.d}/01_env.sh | 0 {nginx => docker/nginx}/conf.d/default.conf | 0 {nginx => docker/nginx}/nginx.conf | 0 {services => docker/services}/nginx.sh | 0 {services => docker/services}/vergilius.sh | 0 9 files changed, 7 insertions(+), 7 deletions(-) rename {consul => docker/consul}/vergilius-443.json (100%) rename {consul => docker/consul}/vergilius-80.json (100%) rename {consul => docker/consul}/vergilius-admin.json (100%) rename {init.d => docker/init.d}/01_env.sh (100%) rename {nginx => docker/nginx}/conf.d/default.conf (100%) rename {nginx => docker/nginx}/nginx.conf (100%) rename {services => docker/services}/nginx.sh (100%) rename {services => docker/services}/vergilius.sh (100%) diff --git a/Dockerfile b/Dockerfile index 5803c3c..5d64eef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,17 +8,17 @@ RUN apt-get install -y ca-certificates nginx git-core python3 build-essential au python3-dev libffi-dev libssl-dev python3-pip dialog nano ENV TERM screen -ADD init.d/01_env.sh /etc/init.d/ -ADD services/nginx.sh /etc/service/nginx/run -ADD services/vergilius.sh /etc/service/vergilius/run +COPY docker/init.d/01_env.sh /etc/init.d/ +COPY docker/services/nginx.sh /etc/service/nginx/run +COPY docker/services/vergilius.sh /etc/service/vergilius/run -COPY consul/* /etc/consul/conf.d/ -COPY nginx/conf.d/*.conf /etc/nginx/conf.d/ -COPY nginx/nginx.conf /etc/nginx/nginx.conf +COPY docker/consul/* /etc/consul/conf.d/ +COPY docker/nginx/conf.d/*.conf /etc/nginx/conf.d/ +COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf RUN rm /etc/nginx/sites-enabled/* && mkdir -p /etc/nginx/sites-enabled/certs && \ mkdir -p /data/dummy_ca/domains/ -ADD src /opt/vergilius +COPY src /opt/vergilius RUN cd /opt/vergilius/ && python3 setup.py install WORKDIR /opt/vergilius/ diff --git a/consul/vergilius-443.json b/docker/consul/vergilius-443.json similarity index 100% rename from consul/vergilius-443.json rename to docker/consul/vergilius-443.json diff --git a/consul/vergilius-80.json b/docker/consul/vergilius-80.json similarity index 100% rename from consul/vergilius-80.json rename to docker/consul/vergilius-80.json diff --git a/consul/vergilius-admin.json b/docker/consul/vergilius-admin.json similarity index 100% rename from consul/vergilius-admin.json rename to docker/consul/vergilius-admin.json diff --git a/init.d/01_env.sh b/docker/init.d/01_env.sh similarity index 100% rename from init.d/01_env.sh rename to docker/init.d/01_env.sh diff --git a/nginx/conf.d/default.conf b/docker/nginx/conf.d/default.conf similarity index 100% rename from nginx/conf.d/default.conf rename to docker/nginx/conf.d/default.conf diff --git a/nginx/nginx.conf b/docker/nginx/nginx.conf similarity index 100% rename from nginx/nginx.conf rename to docker/nginx/nginx.conf diff --git a/services/nginx.sh b/docker/services/nginx.sh similarity index 100% rename from services/nginx.sh rename to docker/services/nginx.sh diff --git a/services/vergilius.sh b/docker/services/vergilius.sh similarity index 100% rename from services/vergilius.sh rename to docker/services/vergilius.sh From 1326d0138768fe890e806da5ab6b1816133c56d7 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 23 Dec 2016 17:46:45 +0500 Subject: [PATCH 09/36] update requirements --- requirements.txt | 4 ++-- src/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2d779fd..ce1d77c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ zope.component==4.2.2 zope.event==4.1.0 zope.interface==4.1.3 -python-consul==0.4.7 -tornado==4.3 +python-consul==0.7 +tornado==4.4.2 funcsigs==1.0.0 mock==1.3.0 acme==0.9.3 diff --git a/src/setup.py b/src/setup.py index c8e629d..73c6588 100644 --- a/src/setup.py +++ b/src/setup.py @@ -2,8 +2,8 @@ from setuptools import find_packages install_requires = [ - 'python-consul', - 'tornado', + 'python-consul==0.7.0', + 'tornado==4.4.2', 'setuptools>=1.0', 'zope.component', 'zope.interface', From bc6b716de7979da15d7cfd3e6650044393dd79e9 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 23 Dec 2016 20:20:56 +0500 Subject: [PATCH 10/36] slight code refactor --- .gitignore | 1 + src/vergilius/__init__.py | 4 ++ .../components/acme_certificate_provider.py | 55 +++++++++++-------- src/vergilius/loop/service_watcher.py | 4 +- src/vergilius/models/certificate.py | 8 +-- src/vergilius/session.py | 7 ++- 6 files changed, 49 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 4afa3a3..79084c5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.iml .idea/ +/venv-vergilius diff --git a/src/vergilius/__init__.py b/src/vergilius/__init__.py index 3fb3cef..228fce7 100644 --- a/src/vergilius/__init__.py +++ b/src/vergilius/__init__.py @@ -21,6 +21,10 @@ class Vergilius(object): identity = None + session = None + + certificate_provider = None + __instance = None def __new__(cls): diff --git a/src/vergilius/components/acme_certificate_provider.py b/src/vergilius/components/acme_certificate_provider.py index ddd40ea..ee85a43 100644 --- a/src/vergilius/components/acme_certificate_provider.py +++ b/src/vergilius/components/acme_certificate_provider.py @@ -1,8 +1,9 @@ -import os, sys, base64, time, logging +import base64 +import logging from concurrent.futures import ThreadPoolExecutor -from datetime import datetime -import tornado.web, tornado.gen +import tornado.gen +import tornado.web from tornado.httpclient import HTTPClient from vergilius import config from consul.tornado import Consul as TornadoConsul @@ -24,10 +25,13 @@ DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' + class AcmeCertificateProvider(object): tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) private_key = None + _acme = None + acme_key = None def __init__(self, *, session): self.session = session @@ -43,8 +47,9 @@ def make_app(self): def fetch_key(self): index, key_data = self.cc.kv.get('vergilius/acme/private_key') - if (key_data): - self.private_key = serialization.load_pem_private_key(key_data['Value'],password=None,backend=default_backend()) + if key_data: + self.private_key = serialization.load_pem_private_key(key_data['Value'], + password=None, backend=default_backend()) else: self.private_key = rsa.generate_private_key( public_exponent=65537, @@ -63,8 +68,9 @@ def init_acme(self): self._acme = client.Client(DIRECTORY_URL, self.acme_key) try: regr = self._acme.register() - acme.agree_to_tos(self._regr) - except: + self._acme.agree_to_tos(regr) + except Exception as e: + logger.error('acme certificate provider error: %s' % e) pass def get_for_domain(self, domain): @@ -74,7 +80,6 @@ def _b64(b): # get token def _parse_token(authzr): - token = None for c in authzr.body.challenges: json = c.chall.to_partial_json() if json['type'] == 'http-01': @@ -83,7 +88,7 @@ def _parse_token(authzr): def _store_token(token): # put token to consul KV thumbprint = _b64(self.acme_key.thumbprint()) - keyauth = '{0}.{1}'.format(token,thumbprint) + keyauth = '{0}.{1}'.format(token, thumbprint) self.cc.kv.put('vergilius/acme/challenge/%s' % token, keyauth) # request challenges for domain @@ -91,15 +96,15 @@ def _store_token(token): token = _parse_token(authzr) _store_token(token) - challenge = [ x for x in authzr.body.challenges if x.typ == 'http-01' ][0] + challenge = [x for x in authzr.body.challenges if x.typ == 'http-01'][0] response, validation = challenge.response_and_validation(self.acme_key) print('chall uri', challenge.uri) result = self._acme.answer_challenge(challenge, response) print('answer result is ', result) - waitUntil = time.time() + 30 - while time.time() < waitUntil: + wait_until = time.time() + 30 + while time.time() < wait_until: logger.debug('polling...') authzr, authzr_response = self._acme.poll(authzr) if authzr.body.status not in ( @@ -111,38 +116,41 @@ def _store_token(token): return authzr def get_csr(self, domains): + """create certificate request for domains""" private_key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend() ) - firstDomain = domains[0] + first_domain = domains[0] csr = x509.CertificateSigningRequestBuilder().subject_name( x509.Name([ x509.NameAttribute(NameOID.COUNTRY_NAME, 'RU'), x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'Yekaterinburg'), x509.NameAttribute(NameOID.LOCALITY_NAME, 'Yekaterinburg'), x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'devopsftw'), - x509.NameAttribute(NameOID.COMMON_NAME, firstDomain), + x509.NameAttribute(NameOID.COMMON_NAME, first_domain), ]) ).add_extension( x509.SubjectAlternativeName([ x509.DNSName(domain) for domain in domains ]), - critical = False, + critical=False, ).sign(private_key, hashes.SHA256(), default_backend()) csr_openssl = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr.public_bytes(serialization.Encoding.PEM) ) - return (private_key, csr_openssl) + return private_key, csr_openssl def get_authzrs(self, domains): - authzrs = [ self.get_for_domain(domain) for domain in domains ] + """request challenges for each domain""" + authzrs = [self.get_for_domain(domain) for domain in domains] return authzrs - def request_certificate(self, csr, domains, authzrs): + def request_certificate(self, csr, authzrs): + """request certificates for solved challenges""" try: response = self._acme.request_issuance(jose.util.ComparableX509(csr), authzrs) cert_data = HTTPClient().fetch(response.uri).body @@ -156,11 +164,13 @@ def request_certificate(self, csr, domains, authzrs): def query_letsencrypt(self, domains): authzrs = self.get_authzrs(domains) domain_key, csr = self.get_csr(domains) - cert = self.request_certificate(csr, domains, authzrs) - return (domain_key, cert) + cert = self.request_certificate(csr, authzrs) + return domain_key, cert @tornado.gen.coroutine def get_certificate(self, id, domains): + """Get certificate for requested domains""" + logger.debug('get cert for domains %s' % domains) sid = yield self.session.get_sid() logger.debug('sid is %s' % sid) @@ -177,12 +187,13 @@ def get_certificate(self, id, domains): result = { 'private_key': key_str, 'public_key': cert_str, - 'expires' : expires + 'expires': expires } logger.debug('cert result is', result) return result + class AcmeChallengeHandler(tornado.web.RequestHandler): tc = TornadoConsul(host=config.CONSUL_HOST) @@ -193,4 +204,4 @@ def get(self, challenge): if data: self.write(data['Value']) else: - raise tornado.web.HTTPError(404) + raise tornado.web.HTTPError(404) \ No newline at end of file diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop/service_watcher.py index 3e73585..c0c2371 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop/service_watcher.py @@ -22,11 +22,11 @@ def watch_services(self): try: index, data = yield consul_tornado.catalog.services(index, wait=None) self.check_services(data) + except ConsulTimeout: + pass except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except ConsulTimeout: - pass def check_services(self, data): # check if service has any of our tags diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py index fc5b7a1..b551b04 100644 --- a/src/vergilius/models/certificate.py +++ b/src/vergilius/models/certificate.py @@ -40,7 +40,7 @@ def __init__(self, service, domains): @tornado.gen.coroutine def fetch(self, index): index, data = yield self.tc.kv.get('vergilius/certificates/%s/' % self.service.id, index=index, recurse=True) - return (index, data) + return index, data @tornado.gen.coroutine def watch(self): @@ -49,11 +49,11 @@ def watch(self): try: index, data = yield self.fetch(index) yield self.load_keys_from_consul(data) + except ConsulTimeout: + pass except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except ConsulTimeout: - pass @tornado.gen.coroutine def load_keys_from_consul(self, data=None): @@ -103,7 +103,7 @@ def acquire_lock(self): """ Create a lock in consul to prevent certificate request race condition """ - self.lock_session_id = self.cc.session.create(behavior='delete',ttl=10) + self.lock_session_id = self.cc.session.create(behavior='delete', ttl=10) return self.cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) def unlock(self): diff --git a/src/vergilius/session.py b/src/vergilius/session.py index 90bf7b3..4b86742 100644 --- a/src/vergilius/session.py +++ b/src/vergilius/session.py @@ -2,14 +2,17 @@ from vergilius import config -import consul, tornado +import consul +import tornado from consul.tornado import Consul as TornadoConsul from tornado.ioloop import IOLoop from tornado.locks import Event +import tornado.gen logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) + class ConsulSession(object): tc = TornadoConsul(host=config.CONSUL_HOST) cc = consul.Consul(host=config.CONSUL_HOST) @@ -29,7 +32,7 @@ def watch(self): @tornado.gen.coroutine def ensure_session(self): - if self._sid == None: + if self._sid is None: self._sid = yield self.create_session() self._waitSid.set() else: From 37f2292e0e30d67deaf1d1eeda78d51e0c25ac51 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 23 Dec 2016 20:21:16 +0500 Subject: [PATCH 11/36] certificate model: validate through x509 certificate loading --- src/vergilius/models/certificate.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py index b551b04..7e0528a 100644 --- a/src/vergilius/models/certificate.py +++ b/src/vergilius/models/certificate.py @@ -1,14 +1,21 @@ import os import time +from datetime import datetime + from tornado.ioloop import IOLoop from tornado.locks import Event import tornado.gen +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + from consul import Consul, ConsulException from consul.base import Timeout as ConsulTimeout from consul.tornado import Consul as TornadoConsul from vergilius import Vergilius, logger, config + class Certificate(object): tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) @@ -151,19 +158,27 @@ def discard_certificate(self): pass def validate(self): - if int(self.expires) < int(time.time()): + if not len(self.private_key) or not len(self.public_key): + logger.warn('[certificate][%s]: validation error: empty key' % self.service.id) + return False + + try: + serialization.load_pem_private_key(self.private_key, password=None, backend=default_backend()) + except: + logger.warn('[certificate][%s]: private key load error: expired' % self.service.id) + return False + + cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate + if datetime.now() > cert.not_valid_after: logger.warn('[certificate][%s]: validation error: expired' % self.service.id) return False + # TODO: get domain names from cert if self.key_domains != self.serialize_domains(): logger.warn('[certificate][%s]: validation error: domains mismatch: %s != %s' % (self.service.id, self.key_domains, self.serialize_domains())) return False - if not len(self.private_key) or not len(self.public_key): - logger.warn('[certificate][%s]: validation error: empty key' % self.service.id) - return False - return True def __del__(self): From 71b1fcff39d9247a309e596033cb75e44490190c Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 27 Dec 2016 14:43:30 +0500 Subject: [PATCH 12/36] slight refactor --- src/vergilius/__init__.py | 9 +++--- .../components/acme_certificate_provider.py | 28 ++++++++----------- src/vergilius/loop/service_watcher.py | 1 + 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/vergilius/__init__.py b/src/vergilius/__init__.py index 228fce7..b43c904 100644 --- a/src/vergilius/__init__.py +++ b/src/vergilius/__init__.py @@ -6,8 +6,8 @@ from tornado import template import vergilius.config -from .components.dummy_certificate_provider import DummyCertificateProvider -from .components.acme_certificate_provider import AcmeCertificateProvider +from vergilius.components.dummy_certificate_provider import DummyCertificateProvider +from vergilius.components.acme_certificate_provider import AcmeCertificateProvider from vergilius.models.identity import Identity from vergilius.session import ConsulSession @@ -33,8 +33,9 @@ def __new__(cls): Vergilius.__instance.init() return Vergilius.__instance - def instance(): - return Vergilius.__instance + @classmethod + def instance(cls): + return cls.__instance def init(self): self.session = ConsulSession() diff --git a/src/vergilius/components/acme_certificate_provider.py b/src/vergilius/components/acme_certificate_provider.py index ee85a43..ae83ec9 100644 --- a/src/vergilius/components/acme_certificate_provider.py +++ b/src/vergilius/components/acme_certificate_provider.py @@ -29,7 +29,6 @@ class AcmeCertificateProvider(object): tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) - private_key = None _acme = None acme_key = None @@ -48,21 +47,21 @@ def make_app(self): def fetch_key(self): index, key_data = self.cc.kv.get('vergilius/acme/private_key') if key_data: - self.private_key = serialization.load_pem_private_key(key_data['Value'], - password=None, backend=default_backend()) + private_key = serialization.load_pem_private_key(key_data['Value'], + password=None, backend=default_backend()) else: - self.private_key = rsa.generate_private_key( + private_key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend() ) - key_data = self.private_key.private_bytes( + key_data = private_key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm=serialization.NoEncryption() ) self.cc.kv.put('vergilius/acme/private_key', key_data) - self.acme_key = jose.JWKRSA(key=self.private_key) + self.acme_key = jose.JWKRSA(key=private_key) def init_acme(self): self._acme = client.Client(DIRECTORY_URL, self.acme_key) @@ -74,7 +73,6 @@ def init_acme(self): pass def get_for_domain(self, domain): - def _b64(b): return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") @@ -107,8 +105,7 @@ def _store_token(token): while time.time() < wait_until: logger.debug('polling...') authzr, authzr_response = self._acme.poll(authzr) - if authzr.body.status not in ( - messages.STATUS_VALID, messages.STATUS_INVALID): + if authzr.body.status not in (messages.STATUS_VALID, messages.STATUS_INVALID): time.sleep(2) else: break @@ -157,8 +154,8 @@ def request_certificate(self, csr, authzrs): cert = x509.load_der_x509_certificate(cert_data, default_backend()) return cert except messages.Error as error: - print ("This script is doomed to fail as no authorization " - "challenges are ever solved. Error from server: {0}".format(error)) + print("This script is doomed to fail as no authorization " + "challenges are ever solved. Error from server: {0}".format(error)) return None def query_letsencrypt(self, domains): @@ -172,15 +169,13 @@ def get_certificate(self, id, domains): """Get certificate for requested domains""" logger.debug('get cert for domains %s' % domains) - sid = yield self.session.get_sid() - logger.debug('sid is %s' % sid) domain_key, cert = yield thread_pool.submit(self.query_letsencrypt, domains) key_str = domain_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption() + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() ) cert_str = cert.public_bytes(serialization.Encoding.PEM) expires = int(cert.not_valid_after.timestamp()) @@ -190,7 +185,6 @@ def get_certificate(self, id, domains): 'expires': expires } - logger.debug('cert result is', result) return result diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop/service_watcher.py index c0c2371..77ff905 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop/service_watcher.py @@ -7,6 +7,7 @@ from vergilius.models.service import Service from vergilius import consul_tornado, logger + class ServiceWatcher(object): def __init__(self): self.services = {} From 1174131b4149738d27209e4bacf1ed3eedb289b0 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 27 Dec 2016 16:59:22 +0500 Subject: [PATCH 13/36] app.py slight refactor --- src/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 0f9897b..9f3229f 100755 --- a/src/app.py +++ b/src/app.py @@ -3,7 +3,7 @@ import signal import time -import tornado +import tornado.ioloop from vergilius import logger, Vergilius from vergilius.loop.nginx_reloader import NginxReloader @@ -44,7 +44,6 @@ def handle_future(f): raise f.exception() def main(): - io_loop = tornado.ioloop.IOLoop.current(); signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) From c8bc75c2244d108bf286278992b007c8fc016ffa Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 27 Dec 2016 18:14:58 +0500 Subject: [PATCH 14/36] refactor stuff --- src/app.py | 10 +- src/vergilius/__init__.py | 38 +--- src/vergilius/{session.py => base.py} | 15 +- .../acme_certificate_provider.py => cert.py} | 0 src/vergilius/components/__init__.py | 0 .../components/certificate_provider.py | 11 - .../components/dummy_certificate_provider.py | 68 ------- .../{loop/service_watcher.py => loop.py} | 32 ++- src/vergilius/loop/__init__.py | 0 src/vergilius/loop/nginx_reloader.py | 30 --- .../{models/service.py => models.py} | 188 +++++++++++++++++- src/vergilius/models/__init__.py | 0 src/vergilius/models/certificate.py | 186 ----------------- src/vergilius/models/identity.py | 85 -------- 14 files changed, 231 insertions(+), 432 deletions(-) rename src/vergilius/{session.py => base.py} (77%) rename src/vergilius/{components/acme_certificate_provider.py => cert.py} (100%) delete mode 100644 src/vergilius/components/__init__.py delete mode 100644 src/vergilius/components/certificate_provider.py delete mode 100644 src/vergilius/components/dummy_certificate_provider.py rename src/vergilius/{loop/service_watcher.py => loop.py} (66%) delete mode 100644 src/vergilius/loop/__init__.py delete mode 100644 src/vergilius/loop/nginx_reloader.py rename src/vergilius/{models/service.py => models.py} (50%) delete mode 100644 src/vergilius/models/__init__.py delete mode 100644 src/vergilius/models/certificate.py delete mode 100644 src/vergilius/models/identity.py diff --git a/src/app.py b/src/app.py index 9f3229f..5bc3203 100755 --- a/src/app.py +++ b/src/app.py @@ -5,9 +5,8 @@ import time import tornado.ioloop -from vergilius import logger, Vergilius -from vergilius.loop.nginx_reloader import NginxReloader -from vergilius.loop.service_watcher import ServiceWatcher +from vergilius import logger +from vergilius.loop import NginxReloader, ServiceWatcher MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 10 @@ -38,16 +37,17 @@ def sig_handler(sig, frame): logger.warning('Caught signal: %s', sig) tornado.ioloop.IOLoop.instance().add_callback(shutdown) + def handle_future(f): tornado.ioloop.IOLoop.current().stop() - if f.exception() != None: + if f.exception() is not None: raise f.exception() + def main(): signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) - Vergilius().instance() consul_handler = ServiceWatcher().watch_services() nginx_reloader = NginxReloader().nginx_reload() diff --git a/src/vergilius/__init__.py b/src/vergilius/__init__.py index b43c904..e1c09c0 100644 --- a/src/vergilius/__init__.py +++ b/src/vergilius/__init__.py @@ -2,42 +2,18 @@ import os from consul import Consul -from consul import tornado as consul_from_tornado +from consul.tornado import Consul as ConsulTornado from tornado import template import vergilius.config -from vergilius.components.dummy_certificate_provider import DummyCertificateProvider -from vergilius.components.acme_certificate_provider import AcmeCertificateProvider -from vergilius.models.identity import Identity -from vergilius.session import ConsulSession +import vergilius.base +import vergilius.cert logger = logging.getLogger(__name__) template_loader = template.Loader(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')) consul = Consul(host=config.CONSUL_HOST) -consul_tornado = consul_from_tornado.Consul(host=config.CONSUL_HOST) - - -class Vergilius(object): - identity = None - - session = None - - certificate_provider = None - - __instance = None - - def __new__(cls): - if Vergilius.__instance is None: - Vergilius.__instance = object.__new__(cls) - Vergilius.__instance.init() - return Vergilius.__instance - - @classmethod - def instance(cls): - return cls.__instance - - def init(self): - self.session = ConsulSession() - self.identity = Identity() - self.certificate_provider = AcmeCertificateProvider(session=self.session) +consul_tornado = ConsulTornado(host=config.CONSUL_HOST) +session = vergilius.base.ConsulSession() +identity = vergilius.base.Identity() +certificate_provider = vergilius.cert.AcmeCertificateProvider() diff --git a/src/vergilius/session.py b/src/vergilius/base.py similarity index 77% rename from src/vergilius/session.py rename to src/vergilius/base.py index 4b86742..cfec0bb 100644 --- a/src/vergilius/session.py +++ b/src/vergilius/base.py @@ -1,10 +1,8 @@ import logging +import consul -from vergilius import config +from vergilius import consul_tornado -import consul -import tornado -from consul.tornado import Consul as TornadoConsul from tornado.ioloop import IOLoop from tornado.locks import Event import tornado.gen @@ -14,8 +12,6 @@ class ConsulSession(object): - tc = TornadoConsul(host=config.CONSUL_HOST) - cc = consul.Consul(host=config.CONSUL_HOST) _sid = None def __init__(self): @@ -37,15 +33,17 @@ def ensure_session(self): self._waitSid.set() else: try: - yield self.tc.session.renew(self._sid) + yield consul_tornado.session.renew(self._sid) except consul.NotFound: logger.error('session not found, trying to recreate') self._sid = yield self.create_session() + except consul.ConsulException as e: + logger.error('consul exception: %s' % e) return True @tornado.gen.coroutine def create_session(self): - sid = yield self.tc.session.create('vergilius', ttl=10, behavior='delete', lock_delay=0) + sid = yield consul_tornado.session.create('vergilius', ttl=10, behavior='delete', lock_delay=0) logger.debug('session created: %s', sid) return sid @@ -53,3 +51,4 @@ def create_session(self): def get_sid(self): yield self._waitSid.wait() return self._sid + diff --git a/src/vergilius/components/acme_certificate_provider.py b/src/vergilius/cert.py similarity index 100% rename from src/vergilius/components/acme_certificate_provider.py rename to src/vergilius/cert.py diff --git a/src/vergilius/components/__init__.py b/src/vergilius/components/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/vergilius/components/certificate_provider.py b/src/vergilius/components/certificate_provider.py deleted file mode 100644 index 27d23ee..0000000 --- a/src/vergilius/components/certificate_provider.py +++ /dev/null @@ -1,11 +0,0 @@ -import zope.interface - - -class ICertificateProvider(zope.interface.Interface): - def get_certificate(self, id, domains): - """ - :param id: string - :param domains: set - :rtype: object with keys private_key, public_key and expires - """ - pass diff --git a/src/vergilius/components/dummy_certificate_provider.py b/src/vergilius/components/dummy_certificate_provider.py deleted file mode 100644 index 90fbf28..0000000 --- a/src/vergilius/components/dummy_certificate_provider.py +++ /dev/null @@ -1,68 +0,0 @@ -import datetime -import hashlib -import os -import subprocess -import time - -import vergilius -from vergilius.components.certificate_provider import ICertificateProvider - -OPENSSL = '/usr/bin/openssl' -KEY_SIZE = 1024 -DAYS = 3650 -DATA_PATH = os.path.join(vergilius.config.DATA_PATH, 'dummy_ca') -X509_EXTRA_ARGS = ('-passin', 'pass:%s' % vergilius.config.SECRET) - - -def openssl(*args): - cmdline = [OPENSSL] + list(args) - subprocess.check_call(cmdline) - - -def check_paths(): - if not os.path.exists(DATA_PATH): - os.mkdir(DATA_PATH) - - if not os.path.exists(os.path.join(DATA_PATH, 'domains')): - os.mkdir(os.path.join(DATA_PATH, 'domains')) - - -class DummyCertificateProvider(object): - @classmethod - def dfile(cls, id, ext): - return os.path.join(DATA_PATH, 'domains', '%s.%s' % (id, ext)) - - def get_certificate(self, id, domains, keysize=KEY_SIZE, days=DAYS): - """ - :param id: string - :type domains: set - """ - check_paths() - - if not os.path.exists(self.dfile(id, 'key')): - openssl('genrsa', '-out', self.dfile(id, 'key'), str(keysize)) - - config_file = open(self.dfile(id, 'config'), 'w') - config_file.write(vergilius.template_loader.load('ssl.html').generate( - domain=sorted(list(domains))[0], dns_list=domains, email=vergilius.config.EMAIL - )) - config_file.close() - - openssl('req', '-new', '-key', self.dfile(id, 'key'), '-out', self.dfile(id, 'request'), - '-config', self.dfile(id, 'config')) - - openssl('x509', '-req', - '-days', str(days), - '-in', self.dfile(id, 'request'), - '-CA', vergilius.Vergilius.identity.get_certificate_path(), - '-CAkey', vergilius.Vergilius.identity.get_private_key_path(), - '-set_serial', - '0x%s' % hashlib.md5(sorted(list(domains))[0] + str(datetime.datetime.now())).hexdigest(), - '-out', self.dfile(id, 'cert'), - '-extensions', 'v3_req', - '-extfile', self.dfile(id, 'config'), - *X509_EXTRA_ARGS) - - return {'private_key': os.path.join(DATA_PATH, self.dfile(id, 'key')), - 'public_key': os.path.join(DATA_PATH, self.dfile(id, 'cert')), - 'expires': int(time.time()) + DAYS * 24 * 60 * 60} diff --git a/src/vergilius/loop/service_watcher.py b/src/vergilius/loop.py similarity index 66% rename from src/vergilius/loop/service_watcher.py rename to src/vergilius/loop.py index e38413b..72b9cbb 100644 --- a/src/vergilius/loop/service_watcher.py +++ b/src/vergilius/loop.py @@ -1,11 +1,37 @@ +import subprocess + +import vergilius +from vergilius.models.service import Service +from vergilius import consul_tornado, logger + import tornado.gen -from tornado.ioloop import IOLoop +from tornado.locks import Event from consul import ConsulException from consul.base import Timeout as ConsulTimeout -from vergilius.models.service import Service -from vergilius import consul_tornado, logger + +class NginxReloader(object): + nginx_update_event = Event() + + def __init__(self): + pass + + @classmethod + @tornado.gen.coroutine + def nginx_reload(cls): + while True: + yield cls.nginx_update_event.wait() + cls.nginx_update_event.clear() + vergilius.logger.info('[nginx]: reload') + try: + subprocess.check_call([vergilius.config.NGINX_BINARY, '-s', 'reload'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logger.error('failed to reload nginx') + + @classmethod + def queue_reload(cls): + cls.nginx_update_event.set() class ServiceWatcher(object): diff --git a/src/vergilius/loop/__init__.py b/src/vergilius/loop/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/vergilius/loop/nginx_reloader.py b/src/vergilius/loop/nginx_reloader.py deleted file mode 100644 index 37fd1bb..0000000 --- a/src/vergilius/loop/nginx_reloader.py +++ /dev/null @@ -1,30 +0,0 @@ -import subprocess -from consul import tornado -from tornado.ioloop import IOLoop -from tornado.locks import Event - -import vergilius -from vergilius import logger - - -class NginxReloader(object): - nginx_update_event = Event() - - def __init__(self): - pass - - @classmethod - @tornado.gen.coroutine - def nginx_reload(cls): - while True: - yield cls.nginx_update_event.wait() - cls.nginx_update_event.clear() - vergilius.logger.info('[nginx]: reload') - try: - subprocess.check_call([vergilius.config.NGINX_BINARY, '-s', 'reload'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - except subprocess.CalledProcessError: - logger.error('failed to reload nginx') - - @classmethod - def queue_reload(cls): - cls.nginx_update_event.set() diff --git a/src/vergilius/models/service.py b/src/vergilius/models.py similarity index 50% rename from src/vergilius/models/service.py rename to src/vergilius/models.py index 04fa6df..5b365f1 100644 --- a/src/vergilius/models/service.py +++ b/src/vergilius/models.py @@ -3,15 +3,23 @@ import subprocess import tempfile import unicodedata +from datetime import datetime import tornado.gen from tornado.ioloop import IOLoop +from tornado.locks import Event -from consul import ConsulException +from consul import Consul, ConsulException from consul.base import Timeout as ConsulTimeout -from vergilius import config, consul_tornado, consul, logger, template_loader -from vergilius.loop.nginx_reloader import NginxReloader -from vergilius.models.certificate import Certificate +from consul.tornado import Consul as TornadoConsul + +from vergilius import config, consul as cc, consul_tornado as tc, logger, template_loader +from vergilius.loop import NginxReloader + + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend class Service(object): @@ -45,7 +53,7 @@ def watch(self): index = None while True and self.active: try: - index, data = yield consul_tornado.health.service(self.name, index, wait=None, passing=True) + index, data = yield tc.health.service(self.name, index, wait=None, passing=True) yield self.parse_data(data) # okay, got data, now reload yield self.update_config() @@ -202,3 +210,173 @@ def slugify(cls, string): string = string.encode('ascii', 'ignore').decode() string = str(re.sub('[^\w\s-]', '', str(string)).strip().lower()) return re.sub('[-\s]+', '-', string) + + +class Certificate(object): + tc = TornadoConsul(host=config.CONSUL_HOST) + cc = Consul(host=config.CONSUL_HOST) + ready_event = Event() + + def __init__(self, service, domains): + """ + :type domains: set + :type service: Service - service name got from consul + """ + self.expires = 0 + self.service = service + self.domains = sorted(domains) + self.key_domains = '' + self.id = '|'.join(self.domains) + + self.private_key = None + self.public_key = None + + self.active = True + self.lock_session_id = None + + if not os.path.exists(os.path.join(config.NGINX_CONFIG_PATH, 'certs')): + os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) + + IOLoop.instance().add_callback(self.unlock) + IOLoop.instance().spawn_callback(self.watch) + + @tornado.gen.coroutine + def fetch(self, index): + index, data = yield tc.kv.get('vergilius/certificates/%s/' % self.service.id, index=index, recurse=True) + return index, data + + @tornado.gen.coroutine + def watch(self): + index = None + while True and self.active: + try: + index, data = yield self.fetch(index) + yield self.load_keys_from_consul(data) + except ConsulTimeout: + pass + except ConsulException as e: + logger.error('consul error: %s' % e) + yield tornado.gen.sleep(5) + + @tornado.gen.coroutine + def load_keys_from_consul(self, data=None): + if data: + for item in data: + key = item['Key'].replace('vergilius/certificates/%s/' % self.service.id, '') + if hasattr(self, key): + setattr(self, key, item['Value']) + + if not self.validate(): + logger.warn('[certificate][%s]: cant validate existing keys' % self.service.id) + self.discard_certificate() + if not (yield self.request_certificate()): + return False + else: + logger.debug('[certificate][%s]: using existing keys' % self.service.id) + else: + if not (yield self.request_certificate()): + return False + + self.write_certificate_files() + self.ready_event.set() + return True + + def write_certificate_files(self): + key_file = open(self.get_key_path(), 'wb') + key_file.write(self.private_key) + key_file.close() + + pem_file = open(self.get_cert_path(), 'wb') + pem_file.write(self.public_key) + pem_file.close() + + def delete_certificate_files(self): + if os.path.exists(self.get_key_path()): + os.remove(self.get_key_path()) + if os.path.exists(self.get_cert_path()): + os.remove(self.get_cert_path()) + + def get_key_path(self): + return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.key') + + def get_cert_path(self): + return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.pem') + + def acquire_lock(self): + """ + Create a lock in consul to prevent certificate request race condition + """ + self.lock_session_id = cc.session.create(behavior='delete', ttl=10) + return cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) + + def unlock(self): + if not self.lock_session_id: + return + + cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) + cc.session.destroy(self.lock_session_id) + self.lock_session_id = None + + @tornado.gen.coroutine + def request_certificate(self): + logger.debug('[certificate][%s] Requesting new keys for %s ' % (self.service.name, self.domains)) + + if not self.acquire_lock(): + logger.debug('[certificate][%s] failed to acquire lock for keys generation' % self.service.name) + return False + + try: + data = yield Vergilius.instance().certificate_provider.get_certificate(self.service.id, self.domains) + + self.private_key = data['private_key'] + cc.kv.put('vergilius/certificates/%s/private_key' % self.service.id, self.private_key) + + self.public_key = data['public_key'] + cc.kv.put('vergilius/certificates/%s/public_key' % self.service.id, self.public_key) + + self.expires = data['expires'] + self.key_domains = self.serialize_domains() + logger.debug('write domain %s' % self.key_domains) + cc.kv.put('vergilius/certificates/%s/expires' % self.service.id, str(self.expires)) + cc.kv.put('vergilius/certificates/%s/key_domains' % self.service.id, self.serialize_domains()) + logger.info('[certificate][%s]: got new keys for %s ' % (self.service.name, self.domains)) + self.write_certificate_files() + except Exception as e: + logger.error(e) + raise e + finally: + self.unlock() + + def serialize_domains(self): + return '|'.join(sorted(self.domains)).encode() + + def discard_certificate(self): + pass + + def validate(self): + if not len(self.private_key) or not len(self.public_key): + logger.warn('[certificate][%s]: validation error: empty key' % self.service.id) + return False + + try: + serialization.load_pem_private_key(self.private_key, password=None, backend=default_backend()) + except: + logger.warn('[certificate][%s]: private key load error: expired' % self.service.id) + return False + + cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate + if datetime.now() > cert.not_valid_after: + logger.warn('[certificate][%s]: validation error: expired' % self.service.id) + return False + + # TODO: get domain names from cert + if self.key_domains != self.serialize_domains(): + logger.warn('[certificate][%s]: validation error: domains mismatch: %s != %s' % + (self.service.id, self.key_domains, self.serialize_domains())) + return False + + return True + + def __del__(self): + self.active = False + self.delete_certificate_files() diff --git a/src/vergilius/models/__init__.py b/src/vergilius/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/vergilius/models/certificate.py b/src/vergilius/models/certificate.py deleted file mode 100644 index 7e0528a..0000000 --- a/src/vergilius/models/certificate.py +++ /dev/null @@ -1,186 +0,0 @@ -import os -import time -from datetime import datetime - -from tornado.ioloop import IOLoop -from tornado.locks import Event -import tornado.gen - -from cryptography import x509 -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.backends import default_backend - -from consul import Consul, ConsulException -from consul.base import Timeout as ConsulTimeout -from consul.tornado import Consul as TornadoConsul -from vergilius import Vergilius, logger, config - - -class Certificate(object): - tc = TornadoConsul(host=config.CONSUL_HOST) - cc = Consul(host=config.CONSUL_HOST) - ready_event = Event() - - def __init__(self, service, domains): - """ - :type domains: set - :type service: Service - service name got from consul - """ - self.expires = 0 - self.service = service - self.domains = sorted(domains) - self.key_domains = '' - self.id = '|'.join(self.domains) - - self.private_key = None - self.public_key = None - - self.active = True - self.lock_session_id = None - - if not os.path.exists(os.path.join(config.NGINX_CONFIG_PATH, 'certs')): - os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) - - IOLoop.instance().add_callback(self.unlock) - IOLoop.instance().spawn_callback(self.watch) - - @tornado.gen.coroutine - def fetch(self, index): - index, data = yield self.tc.kv.get('vergilius/certificates/%s/' % self.service.id, index=index, recurse=True) - return index, data - - @tornado.gen.coroutine - def watch(self): - index = None - while True and self.active: - try: - index, data = yield self.fetch(index) - yield self.load_keys_from_consul(data) - except ConsulTimeout: - pass - except ConsulException as e: - logger.error('consul error: %s' % e) - yield tornado.gen.sleep(5) - - @tornado.gen.coroutine - def load_keys_from_consul(self, data=None): - if data: - for item in data: - key = item['Key'].replace('vergilius/certificates/%s/' % self.service.id, '') - if hasattr(self, key): - setattr(self, key, item['Value']) - - if not self.validate(): - logger.warn('[certificate][%s]: cant validate existing keys' % self.service.id) - self.discard_certificate() - if not (yield self.request_certificate()): - return False - else: - logger.debug('[certificate][%s]: using existing keys' % self.service.id) - else: - if not (yield self.request_certificate()): - return False - - self.write_certificate_files() - self.ready_event.set() - return True - - def write_certificate_files(self): - key_file = open(self.get_key_path(), 'wb') - key_file.write(self.private_key) - key_file.close() - - pem_file = open(self.get_cert_path(), 'wb') - pem_file.write(self.public_key) - pem_file.close() - - def delete_certificate_files(self): - if os.path.exists(self.get_key_path()): - os.remove(self.get_key_path()) - if os.path.exists(self.get_cert_path()): - os.remove(self.get_cert_path()) - - def get_key_path(self): - return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.key') - - def get_cert_path(self): - return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.pem') - - def acquire_lock(self): - """ - Create a lock in consul to prevent certificate request race condition - """ - self.lock_session_id = self.cc.session.create(behavior='delete', ttl=10) - return self.cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) - - def unlock(self): - if not self.lock_session_id: - return - - self.cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) - self.cc.session.destroy(self.lock_session_id) - self.lock_session_id = None - - @tornado.gen.coroutine - def request_certificate(self): - logger.debug('[certificate][%s] Requesting new keys for %s ' % (self.service.name, self.domains)) - - if not self.acquire_lock(): - logger.debug('[certificate][%s] failed to acquire lock for keys generation' % self.service.name) - return False - - try: - data = yield Vergilius.instance().certificate_provider.get_certificate(self.service.id, self.domains) - - self.private_key = data['private_key'] - self.cc.kv.put('vergilius/certificates/%s/private_key' % self.service.id, self.private_key) - - self.public_key = data['public_key'] - self.cc.kv.put('vergilius/certificates/%s/public_key' % self.service.id, self.public_key) - - self.expires = data['expires'] - self.key_domains = self.serialize_domains() - logger.debug('write domain %s' % self.key_domains) - self.cc.kv.put('vergilius/certificates/%s/expires' % self.service.id, str(self.expires)) - self.cc.kv.put('vergilius/certificates/%s/key_domains' % self.service.id, self.serialize_domains()) - logger.info('[certificate][%s]: got new keys for %s ' % (self.service.name, self.domains)) - self.write_certificate_files() - except Exception as e: - logger.error(e) - raise e - finally: - self.unlock() - - def serialize_domains(self): - return '|'.join(sorted(self.domains)).encode() - - def discard_certificate(self): - pass - - def validate(self): - if not len(self.private_key) or not len(self.public_key): - logger.warn('[certificate][%s]: validation error: empty key' % self.service.id) - return False - - try: - serialization.load_pem_private_key(self.private_key, password=None, backend=default_backend()) - except: - logger.warn('[certificate][%s]: private key load error: expired' % self.service.id) - return False - - cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate - if datetime.now() > cert.not_valid_after: - logger.warn('[certificate][%s]: validation error: expired' % self.service.id) - return False - - # TODO: get domain names from cert - if self.key_domains != self.serialize_domains(): - logger.warn('[certificate][%s]: validation error: domains mismatch: %s != %s' % - (self.service.id, self.key_domains, self.serialize_domains())) - return False - - return True - - def __del__(self): - self.active = False - self.delete_certificate_files() diff --git a/src/vergilius/models/identity.py b/src/vergilius/models/identity.py deleted file mode 100644 index 0137bfc..0000000 --- a/src/vergilius/models/identity.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -import subprocess -import tempfile - -import vergilius - -IDENTITY_PATH = os.path.join(vergilius.config.DATA_PATH, 'identity') - - -def openssl(*args): - cmdline = [vergilius.config.OPENSSL_BINARY] + list(args) - subprocess.check_call(cmdline) - - -class Identity(object): - """ - Stores private keys, certificate in consul. - Generates new identity if not specified - """ - - def __init__(self): - self.write_files() - - def get_private_key(self): - index, data = vergilius.consul.kv.get('vergilius/identity/private_key') - if not data: - return False - else: - return data['Value'] - - def get_private_key_path(self): - return os.path.join(IDENTITY_PATH, 'identity.key') - - def get_certificate(self): - index, data = vergilius.consul.kv.get('vergilius/identity/certificate') - if not data: - self.generate_certificate() - return self.get_certificate() - else: - return data['Value'] - - def get_certificate_path(self): - return os.path.join(IDENTITY_PATH, 'identity.crt') - - def generate_identity(self): - vergilius.logger.info('[identity]: generating new identity') - - openssl('genrsa', '-des3', '-passout', 'pass:%s' % vergilius.config.SECRET, '-out', self.get_private_key_path(), - '4096') - - private_key_file = open(self.get_private_key_path(), 'r') - vergilius.consul.kv.put('vergilius/identity/private_key', private_key_file.read()) - private_key_file.close() - - self.get_certificate() - - def generate_certificate(self): - ssl_config_file = tempfile.NamedTemporaryFile(delete=False) - ssl_config_file.write(vergilius.template_loader.load('identity.html').generate( - name='vergilius', email=vergilius.config.EMAIL - )) - ssl_config_file.close() - - openssl('req', '-new', '-x509', '-days', '3650', '-key', self.get_private_key_path(), '-out', - self.get_certificate_path(), '-passin', 'pass:%s' % vergilius.config.SECRET, '-config', - ssl_config_file.name) - - certificate_file = open(self.get_certificate_path(), 'r') - vergilius.consul.kv.put('vergilius/identity/certificate', certificate_file.read()) - certificate_file.close() - - def write_files(self): - if not os.path.exists(IDENTITY_PATH): - os.mkdir(IDENTITY_PATH) - - if not self.get_private_key(): - self.generate_identity() - - private_key_file = open(self.get_private_key_path(), 'wb') - private_key_file.write(self.get_private_key()) - private_key_file.close() - - certificate_file = open(self.get_certificate_path(), 'wb') - certificate_file.write(self.get_certificate()) - certificate_file.close() From 82cc36dd9a34a92524f3d5c7f5e5d9af0e15d867 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Wed, 18 Jan 2017 17:28:36 +0500 Subject: [PATCH 15/36] fix cross-imports --- src/app.py | 21 +++++++++----- src/vergilius/__init__.py | 19 ------------ src/vergilius/base.py | 10 ++++--- src/vergilius/cert.py | 8 +++-- src/vergilius/config.py | 4 +-- src/vergilius/loop.py | 31 +++++++++++--------- src/vergilius/models.py | 61 ++++++++++++++++++++++++--------------- 7 files changed, 83 insertions(+), 71 deletions(-) diff --git a/src/app.py b/src/app.py index 5bc3203..ce79e8b 100755 --- a/src/app.py +++ b/src/app.py @@ -1,15 +1,16 @@ #!/usr/bin/python3 import logging import signal - import time -import tornado.ioloop -from vergilius import logger +import tornado.ioloop +import vergilius.base +from vergilius.cert import AcmeCertificateProvider from vergilius.loop import NginxReloader, ServiceWatcher MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 10 +logger = logging.getLogger('vergilius') logger.setLevel(logging.DEBUG) @@ -48,15 +49,21 @@ def main(): signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) - consul_handler = ServiceWatcher().watch_services() - nginx_reloader = NginxReloader().nginx_reload() + app = App() + sw = ServiceWatcher(app) io_loop = tornado.ioloop.IOLoop.current() - io_loop.add_future(consul_handler, handle_future) - io_loop.add_future(nginx_reloader, handle_future) + io_loop.add_future(sw.watch_services(), handle_future) + io_loop.add_future(app.nginx_reloader.reload(), handle_future) io_loop.start() +class App(object): + def __init__(self): + self.session = vergilius.base.ConsulSession() + self.certificate_provider = AcmeCertificateProvider() + self.nginx_reloader = NginxReloader() + if __name__ == '__main__': main() diff --git a/src/vergilius/__init__.py b/src/vergilius/__init__.py index e1c09c0..e69de29 100644 --- a/src/vergilius/__init__.py +++ b/src/vergilius/__init__.py @@ -1,19 +0,0 @@ -import logging -import os - -from consul import Consul -from consul.tornado import Consul as ConsulTornado -from tornado import template - -import vergilius.config -import vergilius.base -import vergilius.cert - -logger = logging.getLogger(__name__) -template_loader = template.Loader(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')) - -consul = Consul(host=config.CONSUL_HOST) -consul_tornado = ConsulTornado(host=config.CONSUL_HOST) -session = vergilius.base.ConsulSession() -identity = vergilius.base.Identity() -certificate_provider = vergilius.cert.AcmeCertificateProvider() diff --git a/src/vergilius/base.py b/src/vergilius/base.py index cfec0bb..1ad842d 100644 --- a/src/vergilius/base.py +++ b/src/vergilius/base.py @@ -1,14 +1,16 @@ import logging import consul - -from vergilius import consul_tornado +from consul.tornado import Consul as TornadoConsul from tornado.ioloop import IOLoop from tornado.locks import Event import tornado.gen +from vergilius import config + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) +tc = TornadoConsul(host=config.CONSUL_HOST) class ConsulSession(object): @@ -33,7 +35,7 @@ def ensure_session(self): self._waitSid.set() else: try: - yield consul_tornado.session.renew(self._sid) + yield tc.session.renew(self._sid) except consul.NotFound: logger.error('session not found, trying to recreate') self._sid = yield self.create_session() @@ -43,7 +45,7 @@ def ensure_session(self): @tornado.gen.coroutine def create_session(self): - sid = yield consul_tornado.session.create('vergilius', ttl=10, behavior='delete', lock_delay=0) + sid = yield tc.session.create('vergilius', ttl=10, behavior='delete', lock_delay=0) logger.debug('session created: %s', sid) return sid diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index ae83ec9..aa5446a 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -32,8 +32,7 @@ class AcmeCertificateProvider(object): _acme = None acme_key = None - def __init__(self, *, session): - self.session = session + def __init__(self): self.app = self.make_app() self.app.listen(8888) self.fetch_key() @@ -165,13 +164,16 @@ def query_letsencrypt(self, domains): return domain_key, cert @tornado.gen.coroutine - def get_certificate(self, id, domains): + def get_certificate(self, domains): """Get certificate for requested domains""" logger.debug('get cert for domains %s' % domains) domain_key, cert = yield thread_pool.submit(self.query_letsencrypt, domains) + if cert is None: + return None + key_str = domain_key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, diff --git a/src/vergilius/config.py b/src/vergilius/config.py index 0dda231..92e753b 100644 --- a/src/vergilius/config.py +++ b/src/vergilius/config.py @@ -11,10 +11,10 @@ ACME_DIRECTORY_URL = os.environ.get('ACME_DIRECTORY_URL', 'https://acme-staging.api.letsencrypt.org/directory') -OPENSSL_BINARY = os.environ.get('OPENSSL_BINARY', '/usr/bin/openssl') - EMAIL = os.environ.get('EMAIL', 'root@localhost') SECRET = os.environ.get('SECRET') +TEMPLATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates') + if not SECRET: raise Exception('No secret specified!') diff --git a/src/vergilius/loop.py b/src/vergilius/loop.py index 72b9cbb..eeb9b4b 100644 --- a/src/vergilius/loop.py +++ b/src/vergilius/loop.py @@ -1,14 +1,18 @@ +import logging import subprocess -import vergilius -from vergilius.models.service import Service -from vergilius import consul_tornado, logger +from consul.tornado import Consul as TornadoConsul +from consul import ConsulException +from consul.base import Timeout as ConsulTimeout + +from vergilius import config +from vergilius.models import Service import tornado.gen from tornado.locks import Event -from consul import ConsulException -from consul.base import Timeout as ConsulTimeout +logger = logging.getLogger(__name__) +tc = TornadoConsul(host=config.CONSUL_HOST) class NginxReloader(object): @@ -19,15 +23,15 @@ def __init__(self): @classmethod @tornado.gen.coroutine - def nginx_reload(cls): + def reload(cls): while True: yield cls.nginx_update_event.wait() cls.nginx_update_event.clear() - vergilius.logger.info('[nginx]: reload') + logger.info('nginx reload') try: - subprocess.check_call([vergilius.config.NGINX_BINARY, '-s', 'reload'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - except subprocess.CalledProcessError: - logger.error('failed to reload nginx') + subprocess.check_call([config.NGINX_BINARY, '-s', 'reload'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError as e: + logger.error('nginx reload fail. stderr: %s' % e.stderr) @classmethod def queue_reload(cls): @@ -35,18 +39,19 @@ def queue_reload(cls): class ServiceWatcher(object): - def __init__(self): + def __init__(self, app): self.services = {} self.data = {} self.modified = False + self.app = app @tornado.gen.coroutine def watch_services(self): index = None while True: try: - index, data = yield consul_tornado.catalog.services(index, wait=None) + index, data = yield tc.catalog.services(index, wait=None) self.check_services(data) except ConsulTimeout: pass @@ -60,7 +65,7 @@ def check_services(self, data): for service_name in services_to_publish: if service_name not in self.services: logger.info('[service watcher]: new service: %s' % service_name) - self.services[service_name] = Service(service_name) + self.services[service_name] = Service(service_name, self.app) # cleanup stale services for service_name in self.services.keys(): diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 5b365f1..20bb5b5 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -1,4 +1,5 @@ import os +import logging import re import subprocess import tempfile @@ -6,6 +7,7 @@ from datetime import datetime import tornado.gen +import tornado.template from tornado.ioloop import IOLoop from tornado.locks import Event @@ -13,19 +15,21 @@ from consul.base import Timeout as ConsulTimeout from consul.tornado import Consul as TornadoConsul -from vergilius import config, consul as cc, consul_tornado as tc, logger, template_loader -from vergilius.loop import NginxReloader - +from vergilius import config from cryptography import x509 from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend +logger = logging.getLogger(__name__) +template_loader = tornado.template.Loader(config.TEMPLATE_PATH) +tc = TornadoConsul(host=config.CONSUL_HOST) +cc = Consul(host=config.CONSUL_HOST) class Service(object): active = False - def __init__(self, name): + def __init__(self, name, app): """ :type name: unicode - service name got from consul """ @@ -41,6 +45,7 @@ def __init__(self, name): self.active = True self.certificate = None + self.app = app if not os.path.exists(config.NGINX_CONFIG_PATH): os.mkdir(config.NGINX_CONFIG_PATH) @@ -57,16 +62,15 @@ def watch(self): yield self.parse_data(data) # okay, got data, now reload yield self.update_config() + except ConsulTimeout: + pass except ConsulException as e: logger.error('consul error: %s' % e) yield tornado.gen.sleep(5) - except ConsulTimeout: - pass @tornado.gen.coroutine def parse_data(self, data): """ - :type data: set[] """ for protocol in self.domains.keys(): @@ -76,11 +80,11 @@ def parse_data(self, data): self.nodes = {} for node in data: if not node[u'Service'][u'Port']: - logger.warn('[service][%s]: Node %s is ignored due no ServicePort' % (self.id, node[u'Node'])) + logger.warning('[service][%s]: Node %s is ignored due no ServicePort' % (self.id, node[u'Node'])) continue if node[u'Service'][u'Tags'] is None: - logger.warn('[service][%s]: Node %s is ignored due no ServiceTags' % (self.id, node[u'Node'])) + logger.warning('[service][%s]: Node %s is ignored due no ServiceTags' % (self.id, node[u'Node'])) continue self.nodes[node['Node']['Node']] = { @@ -110,7 +114,8 @@ def update_config(self): if not self.certificate: logger.debug('[service][%s] flush stub config' % self.id) self.flush_nginx_config(self.get_stub_config()) - self.certificate = Certificate(service=self, domains=self.domains[u'http2']) + self.certificate = Certificate(service=self, domains=self.domains[u'http2'], + certificate_provider=self.app.certificate_provider) logger.debug('[service][%s] wait for cert' % self.id) yield self.certificate.ready_event.wait() logger.debug('[service][%s] load real https config' % self.id) @@ -145,7 +150,7 @@ def flush_nginx_config(self, nginx_config): config_file.write(nginx_config) config_file.close() logger.info('[service][%s]: got new nginx config %s' % (self.name, self.get_nginx_config_path())) - NginxReloader.queue_reload() + self.app.nginx_reloader.queue_reload() def get_nginx_config_path(self): return os.path.join(config.NGINX_CONFIG_PATH, self.id + '.conf') @@ -173,7 +178,11 @@ def validate(self, config_str): nginx_config_file.close() try: - return_code = subprocess.check_call([config.NGINX_BINARY, '-t', '-c', nginx_config_file.name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return_code = subprocess.check_call( + [config.NGINX_BINARY, '-t', '-c', nginx_config_file.name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) except subprocess.CalledProcessError: return_code = 1 finally: @@ -217,7 +226,7 @@ class Certificate(object): cc = Consul(host=config.CONSUL_HOST) ready_event = Event() - def __init__(self, service, domains): + def __init__(self, service, domains, certificate_provider=None): """ :type domains: set :type service: Service - service name got from consul @@ -228,12 +237,14 @@ def __init__(self, service, domains): self.key_domains = '' self.id = '|'.join(self.domains) - self.private_key = None - self.public_key = None + self.private_key = '' + self.public_key = '' self.active = True self.lock_session_id = None + self.certificate_provider = certificate_provider + if not os.path.exists(os.path.join(config.NGINX_CONFIG_PATH, 'certs')): os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) @@ -267,7 +278,7 @@ def load_keys_from_consul(self, data=None): setattr(self, key, item['Value']) if not self.validate(): - logger.warn('[certificate][%s]: cant validate existing keys' % self.service.id) + logger.warning('[certificate][%s]: cant validate existing keys' % self.service.id) self.discard_certificate() if not (yield self.request_certificate()): return False @@ -326,7 +337,11 @@ def request_certificate(self): return False try: - data = yield Vergilius.instance().certificate_provider.get_certificate(self.service.id, self.domains) + data = yield self.certificate_provider.get_certificate(self.domains) + + if data is None: + logger.error('certificate get failed for service %s' % self.service.name) + return False self.private_key = data['private_key'] cc.kv.put('vergilius/certificates/%s/private_key' % self.service.id, self.private_key) @@ -355,24 +370,24 @@ def discard_certificate(self): def validate(self): if not len(self.private_key) or not len(self.public_key): - logger.warn('[certificate][%s]: validation error: empty key' % self.service.id) + logger.warning('[certificate][%s]: validation error: empty key' % self.service.id) return False try: serialization.load_pem_private_key(self.private_key, password=None, backend=default_backend()) except: - logger.warn('[certificate][%s]: private key load error: expired' % self.service.id) + logger.warning('[certificate][%s]: private key load error: expired' % self.service.id) return False - cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate + cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate if datetime.now() > cert.not_valid_after: - logger.warn('[certificate][%s]: validation error: expired' % self.service.id) + logger.warning('[certificate][%s]: validation error: expired' % self.service.id) return False # TODO: get domain names from cert if self.key_domains != self.serialize_domains(): - logger.warn('[certificate][%s]: validation error: domains mismatch: %s != %s' % - (self.service.id, self.key_domains, self.serialize_domains())) + logger.warning('[certificate][%s]: validation error: domains mismatch: %s != %s' % + (self.service.id, self.key_domains, self.serialize_domains())) return False return True From 6da82e5c0a72d1cc0d8cf739b04641fcfe2dfbe3 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 14:02:16 +0500 Subject: [PATCH 16/36] vergilius small refactor --- src/app.py | 5 +++-- src/vergilius/models.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index ce79e8b..e3da423 100755 --- a/src/app.py +++ b/src/app.py @@ -10,7 +10,8 @@ MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 10 -logger = logging.getLogger('vergilius') +logging.basicConfig(format='%(asctime)s %(levelname)s:%(name)s %(message)s') +logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -34,7 +35,7 @@ def stop_loop(): stop_loop() -def sig_handler(sig, frame): +def sig_handler(sig): logger.warning('Caught signal: %s', sig) tornado.ioloop.IOLoop.instance().add_callback(shutdown) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 20bb5b5..55352ff 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -26,6 +26,7 @@ tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) + class Service(object): active = False @@ -237,8 +238,8 @@ def __init__(self, service, domains, certificate_provider=None): self.key_domains = '' self.id = '|'.join(self.domains) - self.private_key = '' - self.public_key = '' + self.private_key = None + self.public_key = None self.active = True self.lock_session_id = None @@ -369,7 +370,7 @@ def discard_certificate(self): pass def validate(self): - if not len(self.private_key) or not len(self.public_key): + if not self.private_key or not self.public_key: logger.warning('[certificate][%s]: validation error: empty key' % self.service.id) return False From a40567740ae65230c8c70d4eab2cc9f961344faf Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 16:25:12 +0500 Subject: [PATCH 17/36] update tests --- src/vergilius/base.py | 3 +-- src/vergilius/cert.py | 8 +++++++- tests/base_test.py | 26 +++++++++++++++++------- tests/test_certificate.py | 8 ++++---- tests/test_dummy_certificate_provider.py | 4 ++-- tests/test_service.py | 8 +++++--- tests/test_service_watcher.py | 12 ++++++----- 7 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/vergilius/base.py b/src/vergilius/base.py index 1ad842d..8d7719a 100644 --- a/src/vergilius/base.py +++ b/src/vergilius/base.py @@ -52,5 +52,4 @@ def create_session(self): @tornado.gen.coroutine def get_sid(self): yield self._waitSid.wait() - return self._sid - + return self._sid \ No newline at end of file diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index aa5446a..7ea3e16 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -200,4 +200,10 @@ def get(self, challenge): if data: self.write(data['Value']) else: - raise tornado.web.HTTPError(404) \ No newline at end of file + raise tornado.web.HTTPError(404) + + +class DummyCertificateProvider(object): + @tornado.gen.coroutine + def get_certificate(self, domains): + return None diff --git a/tests/base_test.py b/tests/base_test.py index 9ecca6e..85d179d 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -7,28 +7,40 @@ import shutil import tornado -import vergilius -from vergilius import consul, logger -from vergilius.loop.service_watcher import ServiceWatcher -from vergilius.models.identity import Identity +os.environ.setdefault('SECRET', 'test') +import vergilius.base +import vergilius.cert +import vergilius.loop + +from consul import Consul out_hdlr = logging.StreamHandler(sys.stdout) out_hdlr.setFormatter(logging.Formatter('%(asctime)s %(message)s')) out_hdlr.setLevel(logging.DEBUG) +logger = logging.getLogger('tests') logger.addHandler(out_hdlr) logger.setLevel(logging.DEBUG) +cc = Consul() + def start_tornado(): tornado.ioloop.IOLoop.instance().start() +class MockApp(object): + def __init__(self): + self.session = vergilius.base.ConsulSession() + self.certificate_provider = vergilius.cert.DummyCertificateProvider() + self.nginx_reloader = vergilius.loop.NginxReloader() + class BaseTest(unittest.TestCase): @classmethod def setUpClass(cls): super(BaseTest, cls).setUpClass() - cls.watcher = ServiceWatcher() + app = MockApp() + cls.watcher = vergilius.loop.ServiceWatcher(app) cls.watcher.watch_services() threading.Thread(target=start_tornado).start() @@ -40,7 +52,7 @@ def tearDownClass(cls): def setUp(self): super(BaseTest, self).setUp() - consul.kv.delete('vergilius', True) + cc.kv.delete('vergilius', True) try: os.mkdir(vergilius.config.DATA_PATH) @@ -52,7 +64,7 @@ def setUp(self): def tearDown(self): super(BaseTest, self).tearDown() - consul.kv.delete('vergilius', True) + cc.kv.delete('vergilius', True) shutil.rmtree(vergilius.config.NGINX_CONFIG_PATH) shutil.rmtree(vergilius.config.DATA_PATH) diff --git a/tests/test_certificate.py b/tests/test_certificate.py index 710fe4c..37d35c2 100644 --- a/tests/test_certificate.py +++ b/tests/test_certificate.py @@ -1,10 +1,10 @@ from mock import mock +from consul import Consul from base_test import BaseTest -from vergilius import consul -from vergilius.models.certificate import Certificate -from vergilius.models.service import Service +from vergilius.models import Certificate, Service +cc = Consul() class Test(BaseTest): def __init__(self, methodName='runTest'): @@ -13,7 +13,7 @@ def __init__(self, methodName='runTest'): def setUp(self): super(Test, self).setUp() - consul.kv.delete('vergilius', True) + cc.kv.delete('vergilius', True) def test_keys_request(self): cert = Certificate(service=self.service, domains={'example.com'}) diff --git a/tests/test_dummy_certificate_provider.py b/tests/test_dummy_certificate_provider.py index 8f5d635..9175766 100644 --- a/tests/test_dummy_certificate_provider.py +++ b/tests/test_dummy_certificate_provider.py @@ -1,9 +1,9 @@ from base_test import BaseTest -from vergilius import DummyCertificateProvider +from vergilius.cert import DummyCertificateProvider provider = DummyCertificateProvider() class Test(BaseTest): def test_base(self): - provider.get_certificate(id='example.com|foo.example.com', domains={'example.com', 'foo.example.com'}) + provider.get_certificate(domains={'example.com', 'foo.example.com'}) diff --git a/tests/test_service.py b/tests/test_service.py index 19ccff1..73c23cc 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,13 +1,15 @@ from base_test import BaseTest -from vergilius import consul -from vergilius.models.service import Service +from vergilius.models import Service +from consul import Consul + +cc = Consul() class Test(BaseTest): def setUp(self): super(Test, self).setUp() - consul.kv.delete('vergilius', True) + cc.kv.delete('vergilius', True) def test_watcher(self): pass diff --git a/tests/test_service_watcher.py b/tests/test_service_watcher.py index 2fb4107..738cdbe 100644 --- a/tests/test_service_watcher.py +++ b/tests/test_service_watcher.py @@ -1,23 +1,25 @@ import time from base_test import BaseTest -from vergilius import consul +from consul import Consul + +cc = Consul() class Test(BaseTest): def test_poll(self): - consul.agent.service.register('test', 'test', tags=['http'], port=80) + cc.agent.service.register('test', 'test', tags=['http'], port=80) time.sleep(2) self.assertTrue('test' in self.watcher.services, 'service registered') - consul.agent.service.deregister('test') + cc.agent.service.deregister('test') time.sleep(1) self.assertFalse('test' in self.watcher.services.keys(), 'service unregistered') def test_empty_service(self): - consul.agent.service.register('test', 'test') + cc.agent.service.register('test', 'test') time.sleep(2) self.assertFalse('test' in self.watcher.services, 'service not registered') def tearDown(self): - consul.agent.service.deregister('test') + cc.agent.service.deregister('test') From 745fa0c347e60c1f7e2284f9112ba2d33337756d Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 16:43:14 +0500 Subject: [PATCH 18/36] update tests --- tests/base_test.py | 5 ++--- tests/test_service.py | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index 85d179d..607ea84 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -35,11 +35,12 @@ def __init__(self): self.certificate_provider = vergilius.cert.DummyCertificateProvider() self.nginx_reloader = vergilius.loop.NginxReloader() + class BaseTest(unittest.TestCase): @classmethod def setUpClass(cls): super(BaseTest, cls).setUpClass() - app = MockApp() + cls.app = MockApp() cls.watcher = vergilius.loop.ServiceWatcher(app) cls.watcher.watch_services() @@ -60,8 +61,6 @@ def setUp(self): except OSError as e: print(e) - vergilius.Vergilius.init() - def tearDown(self): super(BaseTest, self).tearDown() cc.kv.delete('vergilius', True) diff --git a/tests/test_service.py b/tests/test_service.py index 73c23cc..6ab55ce 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -15,7 +15,7 @@ def test_watcher(self): pass def test_base(self): - service = Service(name='test service') + service = Service(name='test service', app=self.app) service.flush_nginx_config() config_file = service.get_nginx_config_path() @@ -28,7 +28,7 @@ def test_base(self): open(config_file, 'r') def test_http(self): - service = Service(name='test service') + service = Service(name='test service', app=self.app) service.domains[u'http'] = ('example.com',) @@ -37,12 +37,12 @@ def test_http(self): self.assertTrue(service.validate(), 'nginx config is valid') def test_http2(self): - service = Service(name='test service') + service = Service(name='test service', app=self.app) service.domains[u'http2'] = ('example.com',) self.assertTrue(service.validate(), 'nginx config is valid') def test_upstream_nodes(self): - service = Service(name='test service') + service = Service(name='test service', app=self.app) service.domains[u'http'] = ('example.com',) service.nodes['test_node'] = {'address': '127.0.0.1', 'port': '10000'} self.assertTrue(service.validate(), 'nginx config is valid') From 0594c78a5f2ff1e10b57b245a1aab33cb7cf9046 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 16:52:22 +0500 Subject: [PATCH 19/36] update tests --- src/vergilius/models.py | 5 ++--- tests/test_certificate.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 55352ff..0bd62b4 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -115,8 +115,7 @@ def update_config(self): if not self.certificate: logger.debug('[service][%s] flush stub config' % self.id) self.flush_nginx_config(self.get_stub_config()) - self.certificate = Certificate(service=self, domains=self.domains[u'http2'], - certificate_provider=self.app.certificate_provider) + self.certificate = Certificate(service=self, domains=self.domains[u'http2']) logger.debug('[service][%s] wait for cert' % self.id) yield self.certificate.ready_event.wait() logger.debug('[service][%s] load real https config' % self.id) @@ -244,7 +243,7 @@ def __init__(self, service, domains, certificate_provider=None): self.active = True self.lock_session_id = None - self.certificate_provider = certificate_provider + self.certificate_provider = self.service.app.certificate_provider if not os.path.exists(os.path.join(config.NGINX_CONFIG_PATH, 'certs')): os.mkdir(os.path.join(config.NGINX_CONFIG_PATH, 'certs')) diff --git a/tests/test_certificate.py b/tests/test_certificate.py index 37d35c2..159d5bf 100644 --- a/tests/test_certificate.py +++ b/tests/test_certificate.py @@ -6,10 +6,11 @@ cc = Consul() + class Test(BaseTest): def __init__(self, methodName='runTest'): super(Test, self).__init__(methodName) - self.service = Service('test') + self.service = Service('test', app=self.app) def setUp(self): super(Test, self).setUp() From fc4e77533bfa2568f7355a2ed727828bbdf96234 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 17:04:10 +0500 Subject: [PATCH 20/36] update tests --- src/vergilius/models.py | 2 +- tests/base_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 0bd62b4..b23b8eb 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -226,7 +226,7 @@ class Certificate(object): cc = Consul(host=config.CONSUL_HOST) ready_event = Event() - def __init__(self, service, domains, certificate_provider=None): + def __init__(self, service, domains): """ :type domains: set :type service: Service - service name got from consul diff --git a/tests/base_test.py b/tests/base_test.py index 607ea84..6df76af 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -41,7 +41,7 @@ class BaseTest(unittest.TestCase): def setUpClass(cls): super(BaseTest, cls).setUpClass() cls.app = MockApp() - cls.watcher = vergilius.loop.ServiceWatcher(app) + cls.watcher = vergilius.loop.ServiceWatcher(cls.app) cls.watcher.watch_services() threading.Thread(target=start_tornado).start() From 7323bc1a32eb2989763a082851021a5ae9826f7c Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 17:45:22 +0500 Subject: [PATCH 21/36] update tests --- src/vergilius/models.py | 1 + tests/base_test.py | 3 ++- tests/test_certificate.py | 2 +- tests/test_service.py | 14 +++++++------- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index b23b8eb..01538f4 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -127,6 +127,7 @@ def update_config(self): def get_nginx_config(self): """ Generate nginx config from service attributes + :rtype: bytes """ return template_loader.load('service.html').generate(service=self, config=config) diff --git a/tests/base_test.py b/tests/base_test.py index 6df76af..003fbc6 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -5,12 +5,13 @@ import sys import shutil -import tornado +import tornado.ioloop os.environ.setdefault('SECRET', 'test') import vergilius.base import vergilius.cert import vergilius.loop +import vergilius.config from consul import Consul diff --git a/tests/test_certificate.py b/tests/test_certificate.py index 159d5bf..4ebacb4 100644 --- a/tests/test_certificate.py +++ b/tests/test_certificate.py @@ -10,11 +10,11 @@ class Test(BaseTest): def __init__(self, methodName='runTest'): super(Test, self).__init__(methodName) - self.service = Service('test', app=self.app) def setUp(self): super(Test, self).setUp() cc.kv.delete('vergilius', True) + self.service = Service('test', app=self.app) def test_keys_request(self): cert = Certificate(service=self.service, domains={'example.com'}) diff --git a/tests/test_service.py b/tests/test_service.py index 6ab55ce..d7c8378 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -16,12 +16,12 @@ def test_watcher(self): def test_base(self): service = Service(name='test service', app=self.app) - service.flush_nginx_config() + service.flush_nginx_config(service.get_nginx_config()) config_file = service.get_nginx_config_path() self.assertNotEqual(service.read_nginx_config_file().find('server 127.0.0.1:6666'), -1, 'config written and has backup 503') - self.assertTrue(service.validate(), 'nginx config is valid') + self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') service.delete() with self.assertRaises(IOError): @@ -32,21 +32,21 @@ def test_http(self): service.domains[u'http'] = ('example.com',) - self.assertNotEqual(service.get_nginx_config().find('server_name example.com *.example.com;'), -1, + self.assertNotEqual(service.get_nginx_config().decode().find('server_name example.com *.example.com;'), -1, 'server_name and wildcard present') - self.assertTrue(service.validate(), 'nginx config is valid') + self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') def test_http2(self): service = Service(name='test service', app=self.app) service.domains[u'http2'] = ('example.com',) - self.assertTrue(service.validate(), 'nginx config is valid') + self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') def test_upstream_nodes(self): service = Service(name='test service', app=self.app) service.domains[u'http'] = ('example.com',) service.nodes['test_node'] = {'address': '127.0.0.1', 'port': '10000'} - self.assertTrue(service.validate(), 'nginx config is valid') + self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') - config = service.get_nginx_config() + config = service.get_nginx_config().decode() self.assertNotEqual(config.find('server 127.0.0.1:10000;'), -1, 'upstream node present') self.assertEqual(config.find('server 127.0.0.1:6666'), -1, 'backup node deleted') From 204086f5ab9adc56961c3ca2dbf0f229a897b013 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 19 Jan 2017 19:20:21 +0500 Subject: [PATCH 22/36] make dummy cert for unit testing --- src/vergilius/cert.py | 36 +++++++++++++++++++++++++++++++++++- tests/test_service.py | 3 ++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index 7ea3e16..1dbc64e 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -1,4 +1,5 @@ import base64 +import datetime import logging from concurrent.futures import ThreadPoolExecutor @@ -206,4 +207,37 @@ def get(self, challenge): class DummyCertificateProvider(object): @tornado.gen.coroutine def get_certificate(self, domains): - return None + key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, 'RU'), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'Yekaterinburg'), + x509.NameAttribute(NameOID.LOCALITY_NAME, 'Yekaterinburg'), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'devopsftw'), + x509.NameAttribute(NameOID.COMMON_NAME, domains[0]), + ]) + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=10) + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(domains[0])]), + critical=False + ).sign(key, hashes.SHA256(), default_backend()) + + return { + 'private_key': key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ), + 'public_key': cert.public_bytes(serialization.Encoding.PEM), + 'expires': cert.not_valid_after.timestamp() + } diff --git a/tests/test_service.py b/tests/test_service.py index d7c8378..8ce3b39 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,6 +1,6 @@ from base_test import BaseTest -from vergilius.models import Service +from vergilius.models import Service, Certificate from consul import Consul cc = Consul() @@ -39,6 +39,7 @@ def test_http(self): def test_http2(self): service = Service(name='test service', app=self.app) service.domains[u'http2'] = ('example.com',) + service.certificate = Certificate(service, service.domains) # FIXME: create tornado-async test and wait for cert self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') def test_upstream_nodes(self): From e0b6c5e3abab106634c7859f774384f0c9a5581a Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 20 Jan 2017 15:41:01 +0500 Subject: [PATCH 23/36] refactor tests --- src/vergilius/config.py | 3 -- src/vergilius/loop.py | 23 ++++++++------ tests/base_test.py | 28 +++++------------ ...y_certificate_provider.py => test_cert.py} | 7 ++--- tests/test_certificate.py | 25 ---------------- .../{test_service_watcher.py => test_loop.py} | 28 +++++++++-------- tests/{test_service.py => test_models.py} | 30 +++++++++++++------ 7 files changed, 62 insertions(+), 82 deletions(-) rename tests/{test_dummy_certificate_provider.py => test_cert.py} (57%) delete mode 100644 tests/test_certificate.py rename tests/{test_service_watcher.py => test_loop.py} (53%) rename tests/{test_service.py => test_models.py} (70%) diff --git a/src/vergilius/config.py b/src/vergilius/config.py index 92e753b..2dae91a 100644 --- a/src/vergilius/config.py +++ b/src/vergilius/config.py @@ -15,6 +15,3 @@ SECRET = os.environ.get('SECRET') TEMPLATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates') - -if not SECRET: - raise Exception('No secret specified!') diff --git a/src/vergilius/loop.py b/src/vergilius/loop.py index eeb9b4b..4064064 100644 --- a/src/vergilius/loop.py +++ b/src/vergilius/loop.py @@ -50,14 +50,19 @@ def __init__(self, app): def watch_services(self): index = None while True: - try: - index, data = yield tc.catalog.services(index, wait=None) - self.check_services(data) - except ConsulTimeout: - pass - except ConsulException as e: - logger.error('consul error: %s' % e) - yield tornado.gen.sleep(5) + index = yield self.fetch_services(index) + + @tornado.gen.coroutine + def fetch_services(self, index=None): + try: + index, data = yield tc.catalog.services(index, wait=None) + self.check_services(data) + return index + except ConsulTimeout: + pass + except ConsulException as e: + logger.error('consul error: %s' % e) + yield tornado.gen.sleep(1) def check_services(self, data): # check if service has any of our tags @@ -68,7 +73,7 @@ def check_services(self, data): self.services[service_name] = Service(service_name, self.app) # cleanup stale services - for service_name in self.services.keys(): + for service_name in list(self.services): if service_name not in services_to_publish.keys(): logger.info('[service watcher]: removing stale service: %s' % service_name) del self.services[service_name] diff --git a/tests/base_test.py b/tests/base_test.py index 003fbc6..24bb621 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,11 +1,11 @@ import logging import os -import threading import unittest import sys import shutil import tornado.ioloop +import tornado.testing os.environ.setdefault('SECRET', 'test') import vergilius.base @@ -37,23 +37,10 @@ def __init__(self): self.nginx_reloader = vergilius.loop.NginxReloader() -class BaseTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - super(BaseTest, cls).setUpClass() - cls.app = MockApp() - cls.watcher = vergilius.loop.ServiceWatcher(cls.app) - cls.watcher.watch_services() - - threading.Thread(target=start_tornado).start() - - @classmethod - def tearDownClass(cls): - super(BaseTest, cls).tearDownClass() - tornado.ioloop.IOLoop.instance().stop() - +class BaseAsyncTest(tornado.testing.AsyncTestCase): def setUp(self): - super(BaseTest, self).setUp() + super(BaseAsyncTest, self).setUp() + self.app = MockApp() cc.kv.delete('vergilius', True) try: @@ -63,8 +50,9 @@ def setUp(self): print(e) def tearDown(self): - super(BaseTest, self).tearDown() - cc.kv.delete('vergilius', True) - + super(BaseAsyncTest, self).tearDown() shutil.rmtree(vergilius.config.NGINX_CONFIG_PATH) shutil.rmtree(vergilius.config.DATA_PATH) + + def get_new_ioloop(self): + return tornado.ioloop.IOLoop.instance() \ No newline at end of file diff --git a/tests/test_dummy_certificate_provider.py b/tests/test_cert.py similarity index 57% rename from tests/test_dummy_certificate_provider.py rename to tests/test_cert.py index 9175766..d5cbbe5 100644 --- a/tests/test_dummy_certificate_provider.py +++ b/tests/test_cert.py @@ -1,9 +1,8 @@ -from base_test import BaseTest +import unittest from vergilius.cert import DummyCertificateProvider -provider = DummyCertificateProvider() - -class Test(BaseTest): +class DummyCertificateProviderTest(unittest.TestCase): def test_base(self): + provider = DummyCertificateProvider() provider.get_certificate(domains={'example.com', 'foo.example.com'}) diff --git a/tests/test_certificate.py b/tests/test_certificate.py deleted file mode 100644 index 4ebacb4..0000000 --- a/tests/test_certificate.py +++ /dev/null @@ -1,25 +0,0 @@ -from mock import mock - -from consul import Consul -from base_test import BaseTest -from vergilius.models import Certificate, Service - -cc = Consul() - - -class Test(BaseTest): - def __init__(self, methodName='runTest'): - super(Test, self).__init__(methodName) - - def setUp(self): - super(Test, self).setUp() - cc.kv.delete('vergilius', True) - self.service = Service('test', app=self.app) - - def test_keys_request(self): - cert = Certificate(service=self.service, domains={'example.com'}) - self.assertTrue(cert.validate(), 'got valid keys') - - with mock.patch.object(Certificate, 'request_certificate', return_value={}) as mock_method: - Certificate(service=self.service, domains={'example.com'}) - self.assertFalse(mock_method.called, 'check if existing keys are not requested from provider') diff --git a/tests/test_service_watcher.py b/tests/test_loop.py similarity index 53% rename from tests/test_service_watcher.py rename to tests/test_loop.py index 738cdbe..6dba276 100644 --- a/tests/test_service_watcher.py +++ b/tests/test_loop.py @@ -1,25 +1,29 @@ -import time +import tornado.testing +from base_test import BaseAsyncTest, cc +import vergilius.loop -from base_test import BaseTest -from consul import Consul -cc = Consul() +class ServiceWatcherTest(BaseAsyncTest): + def setUp(self): + super().setUp() + self.watcher = vergilius.loop.ServiceWatcher(self.app) + def tearDown(self): + super(ServiceWatcherTest, self).tearDown() + cc.agent.service.deregister('test') -class Test(BaseTest): + @tornado.testing.gen_test def test_poll(self): cc.agent.service.register('test', 'test', tags=['http'], port=80) - time.sleep(2) + yield self.watcher.fetch_services() self.assertTrue('test' in self.watcher.services, 'service registered') + cc.agent.service.deregister('test') - time.sleep(1) + yield self.watcher.fetch_services() self.assertFalse('test' in self.watcher.services.keys(), 'service unregistered') + @tornado.testing.gen_test def test_empty_service(self): cc.agent.service.register('test', 'test') - - time.sleep(2) + yield self.watcher.fetch_services() self.assertFalse('test' in self.watcher.services, 'service not registered') - - def tearDown(self): - cc.agent.service.deregister('test') diff --git a/tests/test_service.py b/tests/test_models.py similarity index 70% rename from tests/test_service.py rename to tests/test_models.py index 8ce3b39..fe327cb 100644 --- a/tests/test_service.py +++ b/tests/test_models.py @@ -1,18 +1,30 @@ - -from base_test import BaseTest +import tornado.testing +from base_test import BaseAsyncTest, cc from vergilius.models import Service, Certificate -from consul import Consul - -cc = Consul() +from mock import mock -class Test(BaseTest): +class CertificateTest(BaseAsyncTest): def setUp(self): - super(Test, self).setUp() + super().setUp() cc.kv.delete('vergilius', True) - def test_watcher(self): - pass + @tornado.testing.gen_test + def test_keys_request(self): + service = Service('test', app=self.app) + cert = Certificate(service, domains={'example.com'}) + yield cert.ready_event.wait() + self.assertTrue(cert.validate(), 'got valid keys') + + with mock.patch.object(Certificate, 'request_certificate', return_value={}) as mock_method: + Certificate(service=service, domains={'example.com'}) + self.assertFalse(mock_method.called, 'check if existing keys are not requested from provider') + + +class ServiceTest(BaseAsyncTest): + def setUp(self): + super(ServiceTest, self).setUp() + cc.kv.delete('vergilius', True) def test_base(self): service = Service(name='test service', app=self.app) From cc64a659af96c24ff640daa099e1b07d77eb9d84 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 20 Jan 2017 15:45:02 +0500 Subject: [PATCH 24/36] cleanup tests --- tests/base_test.py | 5 ----- tests/test_models.py | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index 24bb621..d7ef039 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -7,7 +7,6 @@ import tornado.ioloop import tornado.testing -os.environ.setdefault('SECRET', 'test') import vergilius.base import vergilius.cert import vergilius.loop @@ -26,10 +25,6 @@ cc = Consul() -def start_tornado(): - tornado.ioloop.IOLoop.instance().start() - - class MockApp(object): def __init__(self): self.session = vergilius.base.ConsulSession() diff --git a/tests/test_models.py b/tests/test_models.py index fe327cb..8b357f6 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -48,10 +48,12 @@ def test_http(self): 'server_name and wildcard present') self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') + @tornado.testing.gen_test def test_http2(self): service = Service(name='test service', app=self.app) service.domains[u'http2'] = ('example.com',) - service.certificate = Certificate(service, service.domains) # FIXME: create tornado-async test and wait for cert + service.certificate = Certificate(service, service.domains) + yield service.certificate.ready_event.wait() self.assertTrue(service.validate(service.get_nginx_config()), 'nginx config is valid') def test_upstream_nodes(self): From ea09f2075c61dbddcbea6e76ac72c5619a249205 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Fri, 20 Jan 2017 17:35:35 +0500 Subject: [PATCH 25/36] dummycert update --- src/vergilius/cert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index 1dbc64e..ab0b2f6 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -228,7 +228,7 @@ def get_certificate(self, domains): ).not_valid_after( datetime.datetime.utcnow() + datetime.timedelta(days=10) ).add_extension( - x509.SubjectAlternativeName([x509.DNSName(domains[0])]), + x509.SubjectAlternativeName([x509.DNSName(domain) for domain in domains]), critical=False ).sign(key, hashes.SHA256(), default_backend()) From 533eaf6fde024a63d658fbfcd09bf23838484edb Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 12:32:20 +0500 Subject: [PATCH 26/36] models.Service: track data change cause index changes on every health check --- src/vergilius/models.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 01538f4..26a1544 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -57,12 +57,15 @@ def __init__(self, name, app): @tornado.gen.coroutine def watch(self): index = None + old_data = None while True and self.active: try: index, data = yield tc.health.service(self.name, index, wait=None, passing=True) - yield self.parse_data(data) - # okay, got data, now reload - yield self.update_config() + if old_data != data: + yield self.parse_data(data) + # okay, got data, now reload + yield self.update_config() + old_data = data except ConsulTimeout: pass except ConsulException as e: From 324dde1a6571fe9961a77344adcb7a1e2e85eef3 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 13:29:31 +0500 Subject: [PATCH 27/36] acmecertprovider slight refactor --- src/vergilius/cert.py | 87 ++++++++++++++++++++++++----------------- src/vergilius/config.py | 5 +++ 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index ab0b2f6..8807236 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -23,12 +23,14 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) thread_pool = ThreadPoolExecutor(4) +tc = TornadoConsul(host=config.CONSUL_HOST) -DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' + +def urlsafe_b64(b): + return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") class AcmeCertificateProvider(object): - tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) _acme = None acme_key = None @@ -39,7 +41,8 @@ def __init__(self): self.fetch_key() self.init_acme() - def make_app(self): + @classmethod + def make_app(cls): return tornado.web.Application([ (r"/.well-known/acme-challenge/(.+)", AcmeChallengeHandler), ]) @@ -64,7 +67,7 @@ def fetch_key(self): self.acme_key = jose.JWKRSA(key=private_key) def init_acme(self): - self._acme = client.Client(DIRECTORY_URL, self.acme_key) + self._acme = client.Client(config.ACME_DIRECTORY_URL, self.acme_key) try: regr = self._acme.register() self._acme.agree_to_tos(regr) @@ -72,48 +75,57 @@ def init_acme(self): logger.error('acme certificate provider error: %s' % e) pass - def get_for_domain(self, domain): - def _b64(b): - return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") - - # get token - def _parse_token(authzr): - for c in authzr.body.challenges: - json = c.chall.to_partial_json() - if json['type'] == 'http-01': - return json['token'] - - def _store_token(token): - # put token to consul KV - thumbprint = _b64(self.acme_key.thumbprint()) - keyauth = '{0}.{1}'.format(token, thumbprint) - self.cc.kv.put('vergilius/acme/challenge/%s' % token, keyauth) - + @classmethod + def parse_token(cls, authzr): + for c in authzr.body.challenges: + json = c.chall.to_partial_json() + if json['type'] == 'http-01': + return json['token'] + + def store_token(self, token): + # put token to consul KV + thumbprint = urlsafe_b64(self.acme_key.thumbprint()) + keyauth = '{0}.{1}'.format(token, thumbprint) + self.cc.kv.put('vergilius/acme/challenge/%s' % token, keyauth) + + def delete_token(self, token): + self.cc.kv.delete('vergilius/acme/challenge/%s' % token) + + def auth_domain(self, domain): + """ + request and solve ACME challenge for domain + :param domain: string + :return: + :rtype:client.messages.AuthorizationResource + """ # request challenges for domain authzr = self._acme.request_domain_challenges(domain) - token = _parse_token(authzr) - _store_token(token) + token = self.parse_token(authzr) + self.store_token(token) challenge = [x for x in authzr.body.challenges if x.typ == 'http-01'][0] response, validation = challenge.response_and_validation(self.acme_key) - print('chall uri', challenge.uri) - - result = self._acme.answer_challenge(challenge, response) - print('answer result is ', result) + self._acme.answer_challenge(challenge, response) wait_until = time.time() + 30 while time.time() < wait_until: - logger.debug('polling...') + logger.debug('polling challenge result for %s' % domain) authzr, authzr_response = self._acme.poll(authzr) + logger.debug('status for %s is %s' % domain, authzr.body.status) if authzr.body.status not in (messages.STATUS_VALID, messages.STATUS_INVALID): time.sleep(2) else: break - logger.debug(authzr) + self.delete_token(token) return authzr - def get_csr(self, domains): - """create certificate request for domains""" + @classmethod + def create_csr(cls, domains): + """ + create private key and csr + :param domains: + :return: tuple with private key and csr in OpenSSL library format + """ private_key = rsa.generate_private_key( public_exponent=65537, key_size=2048, @@ -143,7 +155,7 @@ def get_csr(self, domains): def get_authzrs(self, domains): """request challenges for each domain""" - authzrs = [self.get_for_domain(domain) for domain in domains] + authzrs = [self.auth_domain(domain) for domain in domains] return authzrs def request_certificate(self, csr, authzrs): @@ -159,8 +171,13 @@ def request_certificate(self, csr, authzrs): return None def query_letsencrypt(self, domains): + """ + get and solve letsencrypt challenges, create CSR and request certificate for it + :param domains: + :return: + """ authzrs = self.get_authzrs(domains) - domain_key, csr = self.get_csr(domains) + domain_key, csr = self.create_csr(domains) cert = self.request_certificate(csr, authzrs) return domain_key, cert @@ -192,12 +209,10 @@ def get_certificate(self, domains): class AcmeChallengeHandler(tornado.web.RequestHandler): - tc = TornadoConsul(host=config.CONSUL_HOST) - @tornado.gen.coroutine def get(self, challenge): logger.debug('challenge request: %s' % challenge) - index, data = yield self.tc.kv.get('vergilius/acme/challenge/%s' % challenge) + index, data = yield tc.kv.get('vergilius/acme/challenge/%s' % challenge) if data: self.write(data['Value']) else: diff --git a/src/vergilius/config.py b/src/vergilius/config.py index 2dae91a..311d706 100644 --- a/src/vergilius/config.py +++ b/src/vergilius/config.py @@ -15,3 +15,8 @@ SECRET = os.environ.get('SECRET') TEMPLATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates') + +if os.environ.get('ACME_PRODUCTION', 0): + ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org/directory' +else: + ACME_DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' From 6504d84c380bef3f7355b9e6f8d4d05f14c694a7 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 14:37:39 +0500 Subject: [PATCH 28/36] Service: nodes filtering and compare without checks --- src/vergilius/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 26a1544..14c79ac 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -56,16 +56,17 @@ def __init__(self, name, app): @tornado.gen.coroutine def watch(self): - index = None - old_data = None + index = old_nodes = None while True and self.active: try: index, data = yield tc.health.service(self.name, index, wait=None, passing=True) - if old_data != data: + nodes = sorted([{k:svc[k] for k in svc if k != 'Checks'} for svc in data], + key=lambda x: x['Node']['Node']) + if old_nodes != nodes: + # okay, got data, now parse and reload yield self.parse_data(data) - # okay, got data, now reload yield self.update_config() - old_data = data + old_nodes = nodes except ConsulTimeout: pass except ConsulException as e: From afe8d4b8d081d09f8495731a800e4e92d2b279dc Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 16:19:54 +0500 Subject: [PATCH 29/36] sighandler fix --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index e3da423..5492ed8 100755 --- a/src/app.py +++ b/src/app.py @@ -35,7 +35,7 @@ def stop_loop(): stop_loop() -def sig_handler(sig): +def sig_handler(sig, _): logger.warning('Caught signal: %s', sig) tornado.ioloop.IOLoop.instance().add_callback(shutdown) From d22df13c17071c85e64b0be0ee3a1aeca08840a3 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 16:20:13 +0500 Subject: [PATCH 30/36] service: log nginx stderr on invalid config --- src/vergilius/models.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 14c79ac..092d0b4 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -183,19 +183,18 @@ def validate(self, config_str): nginx_config_file.close() try: - return_code = subprocess.check_call( + cp = subprocess.run( [config.NGINX_BINARY, '-t', '-c', nginx_config_file.name], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + check=True ) - except subprocess.CalledProcessError: - return_code = 1 + except subprocess.CalledProcessError as e: + logger.error('[service][%s] nginx config check failed. stderr: ' % self.id, e.stderr) finally: os.unlink(service_config_file.name) os.unlink('%s.pid' % service_config_file.name) os.unlink(nginx_config_file.name) - - return return_code == 0 + return cp.returncode == 0 def delete(self): """ From d5d21cf89e6cdfda5fabc0250c458aee7e9bdae2 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 16:25:08 +0500 Subject: [PATCH 31/36] log level through config --- src/app.py | 3 ++- src/vergilius/cert.py | 2 +- src/vergilius/config.py | 2 ++ src/vergilius/loop.py | 1 + src/vergilius/models.py | 2 ++ 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 5492ed8..a41ecd3 100755 --- a/src/app.py +++ b/src/app.py @@ -7,12 +7,13 @@ import vergilius.base from vergilius.cert import AcmeCertificateProvider from vergilius.loop import NginxReloader, ServiceWatcher +from vergilius import config MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 10 logging.basicConfig(format='%(asctime)s %(levelname)s:%(name)s %(message)s') logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(config.LOG_LEVEL) def shutdown(): diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index 8807236..9e79277 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -21,7 +21,7 @@ import time logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(config.LOG_LEVEL) thread_pool = ThreadPoolExecutor(4) tc = TornadoConsul(host=config.CONSUL_HOST) diff --git a/src/vergilius/config.py b/src/vergilius/config.py index 311d706..900104e 100644 --- a/src/vergilius/config.py +++ b/src/vergilius/config.py @@ -20,3 +20,5 @@ ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org/directory' else: ACME_DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' + +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG') diff --git a/src/vergilius/loop.py b/src/vergilius/loop.py index 4064064..a0cff4a 100644 --- a/src/vergilius/loop.py +++ b/src/vergilius/loop.py @@ -12,6 +12,7 @@ from tornado.locks import Event logger = logging.getLogger(__name__) +logger.setLevel(config.LOG_LEVEL) tc = TornadoConsul(host=config.CONSUL_HOST) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 092d0b4..03d4eb4 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -22,6 +22,8 @@ from cryptography.hazmat.backends import default_backend logger = logging.getLogger(__name__) +logger.setLevel(config.LOG_LEVEL) + template_loader = tornado.template.Loader(config.TEMPLATE_PATH) tc = TornadoConsul(host=config.CONSUL_HOST) cc = Consul(host=config.CONSUL_HOST) From f28313823246194fc3330c8a54ed1b490231c06a Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Tue, 24 Jan 2017 16:34:51 +0500 Subject: [PATCH 32/36] some small typo fixes --- src/vergilius/base.py | 2 +- src/vergilius/models.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/vergilius/base.py b/src/vergilius/base.py index 8d7719a..5073e80 100644 --- a/src/vergilius/base.py +++ b/src/vergilius/base.py @@ -52,4 +52,4 @@ def create_session(self): @tornado.gen.coroutine def get_sid(self): yield self._waitSid.wait() - return self._sid \ No newline at end of file + return self._sid diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 03d4eb4..19ae0ee 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -62,7 +62,7 @@ def watch(self): while True and self.active: try: index, data = yield tc.health.service(self.name, index, wait=None, passing=True) - nodes = sorted([{k:svc[k] for k in svc if k != 'Checks'} for svc in data], + nodes = sorted([{k: svc[k] for k in svc if k != 'Checks'} for svc in data], key=lambda x: x['Node']['Node']) if old_nodes != nodes: # okay, got data, now parse and reload @@ -184,19 +184,21 @@ def validate(self, config_str): ) nginx_config_file.close() + result = False try: - cp = subprocess.run( + subprocess.run( [config.NGINX_BINARY, '-t', '-c', nginx_config_file.name], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True ) + result = True except subprocess.CalledProcessError as e: logger.error('[service][%s] nginx config check failed. stderr: ' % self.id, e.stderr) finally: os.unlink(service_config_file.name) os.unlink('%s.pid' % service_config_file.name) os.unlink(nginx_config_file.name) - return cp.returncode == 0 + return result def delete(self): """ @@ -382,7 +384,7 @@ def validate(self): try: serialization.load_pem_private_key(self.private_key, password=None, backend=default_backend()) except: - logger.warning('[certificate][%s]: private key load error: expired' % self.service.id) + logger.warning('[certificate][%s]: private key load error: expired' % self.service.id, exc_info=True) return False cert = x509.load_pem_x509_certificate(self.public_key, default_backend()) # type: x509.Certificate From c00c44d19e9e8d2d78b87e937a315c39eef7621d Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 9 Feb 2017 14:33:33 +0500 Subject: [PATCH 33/36] process well-known in real http config --- src/vergilius/templates/service.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vergilius/templates/service.html b/src/vergilius/templates/service.html index 63adc26..41c3c68 100644 --- a/src/vergilius/templates/service.html +++ b/src/vergilius/templates/service.html @@ -68,5 +68,9 @@ return 301 https://$server_name$request_uri; } {% end %} + + location /.well-known/ { + proxy_pass http://127.0.0.1:8888; + } } {% end %} \ No newline at end of file From 81df86b8ae338e934ee937a55e17ae737c8db5c4 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 9 Feb 2017 14:44:33 +0500 Subject: [PATCH 34/36] update locking for certificate --- src/vergilius/base.py | 5 +++-- src/vergilius/models.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/vergilius/base.py b/src/vergilius/base.py index 5073e80..cde5ee5 100644 --- a/src/vergilius/base.py +++ b/src/vergilius/base.py @@ -14,9 +14,8 @@ class ConsulSession(object): - _sid = None - def __init__(self): + self._sid = None self._waitSid = Event() IOLoop.instance().spawn_callback(self.watch) pass @@ -37,8 +36,10 @@ def ensure_session(self): try: yield tc.session.renew(self._sid) except consul.NotFound: + self._waitSid.clear() logger.error('session not found, trying to recreate') self._sid = yield self.create_session() + self._waitSid.set() except consul.ConsulException as e: logger.error('consul exception: %s' % e) return True diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 19ae0ee..b288292 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -321,26 +321,28 @@ def get_key_path(self): def get_cert_path(self): return os.path.join(config.NGINX_CONFIG_PATH, 'certs', self.service.id + '.pem') + @tornado.gen.coroutine def acquire_lock(self): """ Create a lock in consul to prevent certificate request race condition """ - self.lock_session_id = cc.session.create(behavior='delete', ttl=10) - return cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) + self.lock_session_id = yield self.service.app.session.get_sid() + result = yield tc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) + return result + @tornado.gen.coroutine def unlock(self): if not self.lock_session_id: return - cc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) - cc.session.destroy(self.lock_session_id) + yield tc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) self.lock_session_id = None @tornado.gen.coroutine def request_certificate(self): logger.debug('[certificate][%s] Requesting new keys for %s ' % (self.service.name, self.domains)) - if not self.acquire_lock(): + if not (yield self.acquire_lock()): logger.debug('[certificate][%s] failed to acquire lock for keys generation' % self.service.name) return False @@ -368,7 +370,7 @@ def request_certificate(self): logger.error(e) raise e finally: - self.unlock() + yield self.unlock() def serialize_domains(self): return '|'.join(sorted(self.domains)).encode() From 99f74305233711a640517ca087c4ac84118b78a5 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Thu, 9 Feb 2017 16:52:06 +0500 Subject: [PATCH 35/36] fix Certificate ready_event --- src/vergilius/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/vergilius/models.py b/src/vergilius/models.py index b288292..40bba7c 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -30,8 +30,6 @@ class Service(object): - active = False - def __init__(self, name, app): """ :type name: unicode - service name got from consul @@ -230,15 +228,12 @@ def slugify(cls, string): class Certificate(object): - tc = TornadoConsul(host=config.CONSUL_HOST) - cc = Consul(host=config.CONSUL_HOST) - ready_event = Event() - def __init__(self, service, domains): """ :type domains: set :type service: Service - service name got from consul """ + self.ready_event = Event() self.expires = 0 self.service = service self.domains = sorted(domains) From 3a74995ef6604885325962d52d0b97e0086f24f1 Mon Sep 17 00:00:00 2001 From: Alex Salt Date: Mon, 6 Mar 2017 14:14:44 +0100 Subject: [PATCH 36/36] dont reacquire certificate if certificate provider fails with exception --- src/vergilius/cert.py | 1 - src/vergilius/models.py | 19 ++++++++++++------- src/vergilius/templates/service.html | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/vergilius/cert.py b/src/vergilius/cert.py index 9e79277..c4f2d51 100644 --- a/src/vergilius/cert.py +++ b/src/vergilius/cert.py @@ -73,7 +73,6 @@ def init_acme(self): self._acme.agree_to_tos(regr) except Exception as e: logger.error('acme certificate provider error: %s' % e) - pass @classmethod def parse_token(cls, authzr): diff --git a/src/vergilius/models.py b/src/vergilius/models.py index 40bba7c..b84ccf5 100644 --- a/src/vergilius/models.py +++ b/src/vergilius/models.py @@ -234,6 +234,7 @@ def __init__(self, service, domains): :type service: Service - service name got from consul """ self.ready_event = Event() + self.is_valid = False self.expires = 0 self.service = service self.domains = sorted(domains) @@ -280,15 +281,18 @@ def load_keys_from_consul(self, data=None): if hasattr(self, key): setattr(self, key, item['Value']) - if not self.validate(): + if self.validate(): + self.is_valid = True + logger.debug('[certificate][%s]: using existing keys' % self.service.id) + else: logger.warning('[certificate][%s]: cant validate existing keys' % self.service.id) self.discard_certificate() if not (yield self.request_certificate()): + self.ready_event.set() return False - else: - logger.debug('[certificate][%s]: using existing keys' % self.service.id) else: if not (yield self.request_certificate()): + self.ready_event.set() return False self.write_certificate_files() @@ -322,7 +326,7 @@ def acquire_lock(self): Create a lock in consul to prevent certificate request race condition """ self.lock_session_id = yield self.service.app.session.get_sid() - result = yield tc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', acquire=self.lock_session_id) + result = yield tc.kv.put('vergilius/locks/cert/%s' % self.service.id, '', acquire=self.lock_session_id) return result @tornado.gen.coroutine @@ -330,7 +334,7 @@ def unlock(self): if not self.lock_session_id: return - yield tc.kv.put('vergilius/certificates/%s/lock' % self.service.id, '', release=self.lock_session_id) + yield tc.kv.put('vergilius/locks/cert/%s' % self.service.id, '', release=self.lock_session_id) self.lock_session_id = None @tornado.gen.coroutine @@ -361,9 +365,10 @@ def request_certificate(self): cc.kv.put('vergilius/certificates/%s/key_domains' % self.service.id, self.serialize_domains()) logger.info('[certificate][%s]: got new keys for %s ' % (self.service.name, self.domains)) self.write_certificate_files() + self.is_valid = True except Exception as e: - logger.error(e) - raise e + logger.error('[certificate][%s]: certificate request error, discarding: %s' % (self.service.id, e)) + self.is_valid = False finally: yield self.unlock() diff --git a/src/vergilius/templates/service.html b/src/vergilius/templates/service.html index 41c3c68..907c9c2 100644 --- a/src/vergilius/templates/service.html +++ b/src/vergilius/templates/service.html @@ -6,7 +6,7 @@ {% if not service.nodes %}server 127.0.0.1:6666;{% end %} } -{% if len(service.domains['http2']) and service.certificate.private_key and service.certificate.public_key %} +{% if len(service.domains['http2']) and service.certificate.is_valid %} server { server_name{% for domain in service.domains['http2'] %} {{ domain }} *.{{ domain }}{% end %}; listen {{config.NGINX_HTTP2_PORT}} ssl http2;