From 168a081a3e5722c5211f8ec6876ed82543820de3 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Tue, 9 Oct 2018 10:51:59 +0100 Subject: [PATCH 1/3] Switch from PyYAML to ruamel.yaml This is a more modern fork which: 1. Uses YAML 1.3, 2. Has some neat round-tripping functionality, 3. Is regularly maintained, 4. Isn't (as of the time of this commit) using deprecated APIs which will break in the next major Python release. --- dev.py | 2 +- routemaster/cli.py | 2 +- routemaster/config/loader.py | 2 +- routemaster/config/tests/test_loading.py | 2 +- routemaster/tests/test_validation.py | 2 +- setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev.py b/dev.py index f1b32767..f91cf888 100644 --- a/dev.py +++ b/dev.py @@ -4,7 +4,7 @@ Use with `python -i dev.py` for a useful interactive shell. """ -import yaml +from ruamel import yaml from routemaster.db import * # noqa: F403, F401 from routemaster.app import App diff --git a/routemaster/cli.py b/routemaster/cli.py index 5446d87c..93737ca7 100644 --- a/routemaster/cli.py +++ b/routemaster/cli.py @@ -1,7 +1,7 @@ """CLI handling for `routemaster`.""" import logging -import yaml +from ruamel import yaml import click from routemaster.app import App diff --git a/routemaster/config/loader.py b/routemaster/config/loader.py index db0e0883..d056e7b5 100644 --- a/routemaster/config/loader.py +++ b/routemaster/config/loader.py @@ -5,7 +5,7 @@ import datetime from typing import Any, Dict, List, Optional -import yaml +from ruamel import yaml import jsonschema import pkg_resources import jsonschema.exceptions diff --git a/routemaster/config/tests/test_loading.py b/routemaster/config/tests/test_loading.py index d35ce2ba..c527b19b 100644 --- a/routemaster/config/tests/test_loading.py +++ b/routemaster/config/tests/test_loading.py @@ -5,7 +5,7 @@ from pathlib import Path from unittest import mock -import yaml +from ruamel import yaml import pytest from routemaster.config import ( diff --git a/routemaster/tests/test_validation.py b/routemaster/tests/test_validation.py index ff1d7601..e220f0de 100644 --- a/routemaster/tests/test_validation.py +++ b/routemaster/tests/test_validation.py @@ -1,6 +1,6 @@ from pathlib import Path -import yaml +from ruamel import yaml import pytest from routemaster.config import ( diff --git a/setup.py b/setup.py index 5d3d10e7..22fcdfcf 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ install_requires=( 'click', - 'pyyaml', + 'ruamel.yaml', 'jsonschema >=2.6', 'flask', 'psycopg2', From b29b7c78f29256d1511523204d1567ac443327fb Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Tue, 9 Oct 2018 10:53:43 +0100 Subject: [PATCH 2/3] Use `yaml.safe_load` everywhere While the YAML we're loading isn't actually untrusted, we do not have a use case for instantiating arbitrary Python objects from the config files so it's much safer to just not do this. --- dev.py | 2 +- routemaster/cli.py | 2 +- routemaster/config/loader.py | 2 +- routemaster/config/tests/test_loading.py | 4 ++-- routemaster/tests/test_validation.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev.py b/dev.py index f91cf888..d0edc5da 100644 --- a/dev.py +++ b/dev.py @@ -18,7 +18,7 @@ def app_from_config(config_path): By default, will use the example.yaml file. """ with open(config_path, 'r') as f: - config = load_config(yaml.load(f)) + config = load_config(yaml.safe_load(f)) class InteractiveApp(App): """ diff --git a/routemaster/cli.py b/routemaster/cli.py index 93737ca7..a33ba5a4 100644 --- a/routemaster/cli.py +++ b/routemaster/cli.py @@ -29,7 +29,7 @@ def main(ctx, config_file): logging.getLogger('schedule').setLevel(logging.CRITICAL) try: - config = load_config(yaml.load(config_file)) + config = load_config(yaml.safe_load(config_file)) except ConfigError: logger.exception("Configuration Error") click.get_current_context().exit(1) diff --git a/routemaster/config/loader.py b/routemaster/config/loader.py index d056e7b5..9cf3276e 100644 --- a/routemaster/config/loader.py +++ b/routemaster/config/loader.py @@ -71,7 +71,7 @@ def _schema_validate(config: Yaml) -> None: 'routemaster.config', 'schema.yaml', ).decode('utf-8') - schema_yaml = yaml.load(schema_raw) + schema_yaml = yaml.safe_load(schema_raw) try: jsonschema.validate(config, schema_yaml) diff --git a/routemaster/config/tests/test_loading.py b/routemaster/config/tests/test_loading.py index c527b19b..b7014527 100644 --- a/routemaster/config/tests/test_loading.py +++ b/routemaster/config/tests/test_loading.py @@ -37,7 +37,7 @@ def reset_environment(): def yaml_data(name: str): with open(f'test_data/{name}.yaml') as f: - return yaml.load(f) + return yaml.safe_load(f) @contextlib.contextmanager @@ -372,7 +372,7 @@ def test_example_config_loads(): assert example_yaml.exists(), "Example file is missing! (is this test set up correctly?)" - example_config = load_config(yaml.load(example_yaml.read_text())) + example_config = load_config(yaml.safe_load(example_yaml.read_text())) # Some basic assertions that we got the right thing loaded assert list(example_config.state_machines.keys()) == ['user_lifecycle'] diff --git a/routemaster/tests/test_validation.py b/routemaster/tests/test_validation.py index e220f0de..407cef7a 100644 --- a/routemaster/tests/test_validation.py +++ b/routemaster/tests/test_validation.py @@ -216,7 +216,7 @@ def test_example_config_is_valid(app): assert example_yaml.exists(), "Example file is missing! (is this test set up correctly?)" - example_config = load_config(yaml.load(example_yaml.read_text())) + example_config = load_config(yaml.safe_load(example_yaml.read_text())) # quick check that we've loaded the config we expect assert list(example_config.state_machines.keys()) == ['user_lifecycle'] From 8f136a23269272d4ce90d9b2f9c1e75999553655 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Tue, 9 Oct 2018 10:57:18 +0100 Subject: [PATCH 3/3] isort --- routemaster/cli.py | 2 +- routemaster/config/loader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/routemaster/cli.py b/routemaster/cli.py index a33ba5a4..33f642b5 100644 --- a/routemaster/cli.py +++ b/routemaster/cli.py @@ -1,8 +1,8 @@ """CLI handling for `routemaster`.""" import logging -from ruamel import yaml import click +from ruamel import yaml from routemaster.app import App from routemaster.cron import CronThread diff --git a/routemaster/config/loader.py b/routemaster/config/loader.py index 9cf3276e..90d01702 100644 --- a/routemaster/config/loader.py +++ b/routemaster/config/loader.py @@ -5,10 +5,10 @@ import datetime from typing import Any, Dict, List, Optional -from ruamel import yaml import jsonschema import pkg_resources import jsonschema.exceptions +from ruamel import yaml from routemaster.config.model import ( Gate,