diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/.dockerignore b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/.dockerignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/.dockerignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/Dockerfile.{{cookiecutter.PROJECT_NAME_SLUG}} b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/Dockerfile.{{cookiecutter.PROJECT_NAME_SLUG}} index 314b663c..524dc069 100644 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/Dockerfile.{{cookiecutter.PROJECT_NAME_SLUG}} +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/Dockerfile.{{cookiecutter.PROJECT_NAME_SLUG}} @@ -1,24 +1,37 @@ -ARG PYTHON_VERSION=3.7.0-alpine3.8 +ARG PYTHON_VERSION=3.7 -FROM python:${PYTHON_VERSION} as builder +FROM python:${PYTHON_VERSION} AS builder -WORKDIR /usr/src/{{cookiecutter.PROJECT_NAME_SLUG}} +WORKDIR /src + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +RUN pip wheel --no-cache-dir -r requirements.txt -w /deps COPY requirements-dev.txt . RUN pip install --no-cache-dir -r requirements-dev.txt COPY . . -# Run tests, linter -RUN pytest testapp.py \ +RUN flake8 *.py \ && isort --check-only *.py \ - && flake8 *.py + && pydocstyle *.py \ + && python -m doctest *.py + +FROM python:${PYTHON_VERSION}-slim AS runtime + +WORKDIR /{{cookiecutter.PROJECT_NAME_SLUG}}/deps +COPY --from=builder /deps . +RUN pip install --no-cache-dir *.whl + +WORKDIR /{{cookiecutter.PROJECT_NAME_SLUG}}/src +COPY --from=builder /src . -# Release -FROM python:${PYTHON_VERSION} +ENV HOST=0.0.0.0 +ENV PORT=80 +ENV WORKERS=4 +ENV LOG_LEVEL=info -WORKDIR /{{cookiecutter.PROJECT_NAME_SLUG}} -COPY --from=builder /usr/src/{{cookiecutter.PROJECT_NAME_SLUG}} /{{cookiecutter.PROJECT_NAME_SLUG}} -RUN pip install -r requirements.txt +EXPOSE $PORT -CMD ["python", "main.py"] +CMD gunicorn --log-level="$LOG_LEVEL" --bind="$HOST:$PORT" --workers="$WORKERS" "main:build_app(wsgi=True)" diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/api.spec.yaml b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/api.spec.yaml new file mode 100644 index 00000000..0af8d24c --- /dev/null +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/api.spec.yaml @@ -0,0 +1,32 @@ +swagger: '2.0' + +info: + title: Transformation API + version: '1.0' + +basePath: '/' + +paths: + '/': + post: + summary: Transform the data. + operationId: main.main + consumes: + - application/json + parameters: + - $ref: '#/parameters/Data' + responses: + 200: + description: The transformed data. + +parameters: + Data: + name: data + description: The data to transform. + in: body + schema: + $ref: '#/definitions/Data' + required: true + +definitions: +{{DATA_SCHEMA}} diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/data.spec.yaml b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/data.spec.yaml new file mode 100644 index 00000000..aba41e58 --- /dev/null +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/data.spec.yaml @@ -0,0 +1,11 @@ +Data: + properties: + key: + description: The key of the input. + type: string + intValue: + description: The value of the input. + type: number + required: + - key + - intValue diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/datahelper.py b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/datahelper.py deleted file mode 100644 index 32789582..00000000 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/datahelper.py +++ /dev/null @@ -1,36 +0,0 @@ -""" Helper functions for the customer application """ -import json - -from jsonschema import validate - -IN_MEMORY_FILES = {} - - -def validate_schema(data, schema_filepath: str): - """ Validates the input json data against our schema - defined in schema_example.json - - Args: serialized json object that server has retrieved - """ - if schema_filepath is None: - schema_filepath = 'schema_example.json' - if schema_filepath not in IN_MEMORY_FILES.keys(): - with open(schema_filepath, 'r') as schema: - schema_data = schema.read() - sample_schema = json.loads(schema_data) - IN_MEMORY_FILES[schema_filepath] = sample_schema - - parsed_json = json.loads(data) - validate(parsed_json, IN_MEMORY_FILES[schema_filepath]) - - -def transform(data: object): - """ Applies simple transformation to the data for the - final output - - Args: - data: serialized json object that has been validated against schema - """ - parsed_json = json.loads(data) - parsed_json['intValue'] = parsed_json['intValue'] + 100 - return parsed_json diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logging.yaml b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logging.yaml deleted file mode 100644 index 7e0bd4d0..00000000 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logging.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -version: 1 -disable_existing_loggers: False -formatters: - simple: - format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - -handlers: - console: - class: logging.StreamHandler - level: DEBUG - formatter: simple - stream: ext://sys.stdout - -loggers: - my_module: - level: ERROR - handlers: [console] - propagate: no - -root: - level: INFO - handlers: [console] diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logic.py b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logic.py new file mode 100644 index 00000000..32723385 --- /dev/null +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/logic.py @@ -0,0 +1,24 @@ +""" +Implements the business logic for {{cookiecutter.PROJECT_NAME_SLUG}}. + +Overwrite the sample implementation in this file with your own requirements. +The framework expects a function named `transform` to be present in this +module but helper functions may be added at your convenience. +""" + + +def transform(data: dict) -> dict: + """ + Apply the required transformations to the input data. + + The input data for this function is guaranteed to follow the specification + provided to the framework. Consult `data.spec.yaml` for an example. + + Args: + data: input object to transform + + >>> transform({'intValue': 100}) + {'intValue': 200} + """ + data['intValue'] += 100 + return data diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/main.py b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/main.py index 10eaf735..c2705837 100644 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/main.py +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/main.py @@ -1,94 +1,77 @@ -""" Entrypoint for customer application. Listens for HTTP requests from -the input reader, and sends the transformed message to the output writer. """ -import json -import logging -import os -from http.server import BaseHTTPRequestHandler -from http.server import HTTPServer - -import requests - -import datahelper - -# HOST & PORT are the values used to run the current application -HOST = os.getenv("HOST") -PORT = os.getenv("PORT") - -# OUTPUT_URL is the url which receives all the output messages after they are processed by the app -OUTPUT_URL = os.getenv("OUTPUT_URL") - -# Filepath for the JSON schema which represents -# the schema for the expected input messages to the app -SCHEMA_FILEPATH = os.getenv("SCHEMA_FILEPATH") - - -class Socket(BaseHTTPRequestHandler): - """Handles HTTP requests that come to the server.""" - - def _set_headers(self): - """Sets common headers when returning an OK HTTPStatus. """ - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - - def do_POST(self): - """Handles a POST request to the server. - Sends 400 error if there is an issue, otherwise sends a success message. - - Raises: - ValidationError: Returns when data given is not valid against schema - HTTPError: Returns when there is an error sending message to output url - - """ - content_length = int(self.headers['Content-Length']) - data = self.rfile.read(content_length) - data = data.decode("utf-8") - - try: - datahelper.validate_schema(data, SCHEMA_FILEPATH) - except BaseException: - self.send_error( - 400, 'Incorrect data format. Please check JSON schema.') - logging.error('Incorrect data format. Please check JSON schema.') - raise - - try: - transformed_data = datahelper.transform(data) - output_message(transformed_data) - self._set_headers() - self.wfile.write(bytes("Data successfully consumed", 'utf8')) - except BaseException: - self.send_error(400, 'Error when sending output message') - logging.error('Error when sending output message') - raise - - -def output_message(data: object): - """Outputs the transformed payload to the specified HTTP endpoint - - Args: - data: transformed json object to send to output writer - """ - request = requests.post(OUTPUT_URL, data=json.dumps(data)) - if request.status_code != 200: - - logging.error("Error with a request %s and message not sent was %s", - request.status_code, data) - else: - logging.info("%s Response received from output writer", - request.status_code) +""" +Entrypoint for customer application. + +Listens for HTTP(S) requests from the input reader, transforms the message +and sends the transformed message to the output writer. + +You can view a specification for the API at /swagger.json as well as a testing +console at /ui/. + +All business logic for {{cookiecutter.PROJECT_NAME_SLUG}} should be included +in `logic.py` and it is unlikely that changes to this `main.py` file will be +required. +""" +from logging import getLogger + +from requests import post + +import settings +from logic import transform + +LOG = getLogger(__name__) + + +def main(data: dict): + """Transform the input and forward the transformed payload via HTTP(S).""" + data = transform(data) + + if not settings.OUTPUT_URL: + LOG.warning('Not OUTPUT_URL specified, not forwarding data.') + return data, 500 + + response = post(settings.OUTPUT_URL, json=data) + if not response.ok: + LOG.error('Error %d with post to url %s and body %s.', + response.status_code, settings.OUTPUT_URL, data) + return data, response.status_code + LOG.info('Response %d received from output writer.', response.status_code) + return data, 200 -def run(server_class=HTTPServer, handler_class=Socket): - """Run the server on specified host and port, using our - custom Socket class to receive and process requests. - """ - server_address = (HOST, int(PORT)) - httpd = server_class(server_address, handler_class) - logging.info('Running server on host ${HOST} and port ${PORT}') - httpd.serve_forever() +def build_app(wsgi=False): + """Create the web server.""" + from connexion import App + + with open(settings.DATA_SCHEMA, encoding='utf-8') as fobj: + data_schema = '\n'.join(' {}'.format(line.rstrip('\r\n')) + for line in fobj) + + LOG.setLevel(settings.LOG_LEVEL) + + app = App(__name__) + app.add_api('api.spec.yaml', arguments={'DATA_SCHEMA': data_schema}) + + return app.app if wsgi else app + + +def cli(): + """Command line interface for the web server.""" + from argparse import ArgumentParser + + from yaml import YAMLError + + parser = ArgumentParser(description=__doc__) + parser.parse_args() + + try: + app = build_app() + except YAMLError as ex: + parser.error('Unable to parse {} as a Swagger spec.\n\n{}' + .format(settings.DATA_SCHEMA, ex)) + else: + app.run(port=settings.PORT, host=settings.HOST) if __name__ == "__main__": - run() + cli() diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements-dev.txt b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements-dev.txt index a168a295..5662ce21 100644 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements-dev.txt +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements-dev.txt @@ -1,7 +1,3 @@ -hypothesis -python-dotenv -jsonschema -pytest flake8 isort -requests +pydocstyle diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements.txt b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements.txt index c2d74f49..399f4420 100644 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements.txt +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/requirements.txt @@ -1,3 +1,4 @@ -python-dotenv -jsonschema +connexion[swagger-ui] +environs +gunicorn requests diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/schema_example.json b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/schema_example.json deleted file mode 100644 index 0291f909..00000000 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/schema_example.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "object", - "properties": { - "key": {"type": "string"}, - "intValue": {"type": "integer"} - } -} \ No newline at end of file diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/settings.py b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/settings.py new file mode 100644 index 00000000..d7226332 --- /dev/null +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/settings.py @@ -0,0 +1,15 @@ +""" +Module to hold all application configuration. + +All environment variable access hould happen in this module only. +""" +from environs import Env + +env = Env() +env.read_env() + +PORT = env.int('PORT', 8888) +HOST = env('HOST', '127.0.0.1') +DATA_SCHEMA = env('SCHEMA_FILEPATH', 'data.spec.yaml') +OUTPUT_URL = env('OUTPUT_URL', '') +LOG_LEVEL = env('LOG_LEVEL', 'INFO').upper() diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/setup.cfg b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/setup.cfg index e7671d3d..34d8b4d0 100644 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/setup.cfg +++ b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/setup.cfg @@ -4,3 +4,10 @@ max-line-length = 120 [isort] force_single_line = True line_length = 120 + +[pydocstyle] +ignore = + D102, + D104, + D203, + D212 diff --git a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/testapp.py b/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/testapp.py deleted file mode 100644 index e7bff16c..00000000 --- a/agogosml_cli/cli/templates/apps/simple/{{cookiecutter.PROJECT_NAME_SLUG}}/testapp.py +++ /dev/null @@ -1,46 +0,0 @@ -""" Unit tests for the customer app """ -import json -import os -from pathlib import Path - -from dotenv import load_dotenv -from hypothesis import given -from hypothesis.strategies import characters -from hypothesis.strategies import fixed_dictionaries -from hypothesis.strategies import integers -from hypothesis.strategies import text - -import datahelper - -BASE_DIR = Path(__file__).parent.absolute() -load_dotenv(dotenv_path=str(BASE_DIR / ".env")) - -SCHEMA_FILEPATH = os.getenv('SCHEMA_FILEPATH') - -# TO DO: Make Unit Tests More Robust - - -@given( - data=fixed_dictionaries({ - 'key': text(characters()), - 'intValue': integers() - })) -def test_validation(data): - """ Test that the schema validator is working as expected """ - encoded_json = json.dumps(data) - assert datahelper.validate_schema(encoded_json, SCHEMA_FILEPATH) is None - - -@given( - data=fixed_dictionaries({ - 'key': text(characters()), - 'intValue': integers() - })) -def test_transform(data): - """ Test the simple transformation on the input data """ - encoded_original_json = json.dumps(data) - transformed_object = datahelper.transform(encoded_original_json) - - data['intValue'] = data['intValue'] + 100 - - assert transformed_object == data diff --git a/agogosml_cli/tests/test_cli_generate.py b/agogosml_cli/tests/test_cli_generate.py index cd8119e4..344725b6 100644 --- a/agogosml_cli/tests/test_cli_generate.py +++ b/agogosml_cli/tests/test_cli_generate.py @@ -35,14 +35,15 @@ 'testproject/output_writer/Dockerfile.output_writer', 'testproject/output_writer/logging.yaml', 'testproject/output_writer/main.py', + 'testproject/testproject/.dockerignore', + 'testproject/testproject/api.spec.yaml', + 'testproject/testproject/data.spec.yaml', 'testproject/testproject/Dockerfile.testproject', - 'testproject/testproject/logging.yaml', + 'testproject/testproject/logic.py', 'testproject/testproject/main.py', - 'testproject/testproject/requirements-dev.txt', 'testproject/testproject/requirements.txt', - 'testproject/testproject/datahelper.py', - 'testproject/testproject/schema_example.json', - 'testproject/testproject/testapp.py' + 'testproject/testproject/requirements-dev.txt', + 'testproject/testproject/setup.cfg', )