From 6faec6843ebd4991ea29185a2b669ab552ac3563 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Wed, 3 Jun 2020 17:35:17 -0400
Subject: [PATCH 01/11] Create shipment-creator bash function

---
 apps/shipments/serializers/shipment.py |   4 +-
 bin/create_shipments                   |   3 +
 bin/dev-tools/create_shipments.py      | 312 +++++++++++++++++++++++++
 3 files changed, 317 insertions(+), 2 deletions(-)
 create mode 100755 bin/create_shipments
 create mode 100755 bin/dev-tools/create_shipments.py

diff --git a/apps/shipments/serializers/shipment.py b/apps/shipments/serializers/shipment.py
index e53ecddc..2dcb7b5f 100644
--- a/apps/shipments/serializers/shipment.py
+++ b/apps/shipments/serializers/shipment.py
@@ -396,8 +396,8 @@ class ShipmentVaultSerializer(NullableFieldsMixin, serializers.ModelSerializer):
 
     class Meta:
         model = Shipment
-        exclude = ('owner_id', 'storage_credentials_id', 'background_data_hash_interval',
-                   'vault_id', 'vault_uri', 'shipper_wallet_id', 'carrier_wallet_id', 'manual_update_hash_interval',
+        exclude = ('owner_id', 'storage_credentials_id', 'background_data_hash_interval', 'vault_id', 'vault_uri',
+                   'shipper_wallet_id', 'carrier_wallet_id', 'moderator_wallet_id', 'manual_update_hash_interval',
                    'contract_version', 'device', 'updated_by', 'state', 'exception', 'delayed', 'expected_delay_hours',
                    'geofences', 'assignee_id', 'gtx_required', 'gtx_validation', 'gtx_validation_timestamp',
                    'aftership_tracking', 'carrier_abbv', 'arrival_est')
diff --git a/bin/create_shipments b/bin/create_shipments
new file mode 100755
index 00000000..2dc49050
--- /dev/null
+++ b/bin/create_shipments
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+bin/dev-tools/create_shipments.py "$@"
diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
new file mode 100755
index 00000000..833c74a2
--- /dev/null
+++ b/bin/dev-tools/create_shipments.py
@@ -0,0 +1,312 @@
+#! /usr/bin/env python3
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone, timedelta
+from json.decoder import JSONDecodeError
+from random import randint
+from uuid import uuid4
+
+import requests
+from copy import deepcopy
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--total", "-t", help="Total amount of shipments to be created, defaults to 10", type=int)
+parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes", type=int)
+parser.add_argument("--partition", "-p", help="Number shipments to create per wallet, defaults to 10", type=int)
+parser.add_argument("--carrier", "-c", help="Set carrier wallet owner (defaults to user 1)")
+parser.add_argument("--shipper", "-s", help="Set shipper wallet owner (defaults to user 1)")
+parser.add_argument("--moderator", "-m", help="Set moderator wallet owner (defaults to None)")
+parser.add_argument("--verbose", "-v", help="More descriptive when running functions.", action="store_true")
+parser.add_argument("--device", "-d", help="Add devices to shipment creation, defaults to false",
+                    action="store_true")
+parser.add_argument("--attributes", "-a", action='append',
+                    help="Attributes to be sent in shipment creation (defaults to required values)")
+parser.add_argument("--add_tracking", help="Adds tracking data to shipments (requires --device or -d)",
+                    action='store_true')
+parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
+                    action='store_true')
+
+# Keep all but the first command-line arguments
+argument_list = sys.argv[1:]
+
+
+class CreateShipments:
+    # Default variables for local use
+    profiles_url = 'http://localhost:9000'
+    transmission_url = 'http://localhost:8000'
+    client_id = 892633
+    users = {
+        'user_1': {
+            'username': 'user1@shipchain.io',
+            'password': 'user1Password',
+            'token': None,
+            'token_exp': None,
+        }, 'user_2': {
+            'username': 'user2@shipchain.io',
+            'password': 'user2Password',
+            'token': None,
+            'token_exp': None,
+        }
+    }
+    tracking_data = {
+        "position": {
+            "latitude": -81.048253,
+            "longitude": 34.628643,
+            "altitude": 924,
+            "source": "gps",
+            "uncertainty": 95,
+            "speed": 34
+        },
+        "version": "1.2.4",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+    telemetry_data = {
+        "version": "1.2.4",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+    # Values that can be set via command line
+    attributes = {}
+    add_tracking = False
+    add_telemetry = False
+
+    # Values that cannot be set via command line but dynamic within function
+    active_users = {'user_1'}
+    errors = []
+    shipments = []
+
+    def __init__(self):
+        args = parser.parse_args()
+        self.carrier = self._validate_user(args.carrier) if args.carrier else self.users['user_1']
+        self.shipper = self._validate_user(args.shipper) if args.shipper else self.users['user_1']
+        self.moderator = self._validate_user(args.moderator) if args.moderator else None
+        self.sequence_number = args.startnumber if args.startnumber else 0
+        self.total = int(args.total) if args.total else 10
+        self.verbose = args.verbose
+        self.device = args.device
+        self.chunk_size = int(args.partition) if args.partition else 10
+        if args.add_tracking and not self.device:
+            parser.error(f'--add_tracking requires --device or -d')
+        self.add_tracking = args.add_tracking
+        if args.add_telemetry and not self.device:
+            parser.error(f'--add_telemetry requires --device or -d')
+        self.add_telemetry = args.add_telemetry
+        if args.attributes:
+            self._validate_attributes(args.attributes)
+        if self.users['user_1'] not in (self.moderator, self.carrier, self.shipper):
+            self.active_users.remove('user_1')
+
+    def _chunker(self, array):
+        return (array[i:i + self.chunk_size] for i in range(0, len(array), self.chunk_size))
+
+    def _validate_user(self, user):
+        if user not in self.users:
+            self._process_failure(f'Invalid user: {user}')
+        if user not in self.active_users:
+            self.active_users.add(user)
+        return self.users[user]
+
+    def _validate_attributes(self, attributes):
+        print(f'Validating attributes: {attributes}') if self.verbose else None
+        for attribute in attributes:
+            try:
+                attribute = json.loads(attribute)
+            except JSONDecodeError:
+                print(f'Non json attribute recieved: {attribute}') if self.verbose else None
+            if type(attribute) == dict:
+                self.attributes.update(attribute)
+            else:
+                try:
+                    key, value = attribute.split("=")
+                except ValueError:
+                    self._process_failure(f'Invalid format for attribute: {attribute}, should be in format: key=value')
+                self.attributes[key] = value
+
+    def _process_failure(self, error_message):
+        print(f'Shipment Creator failed: {error_message}')
+        print(f'Response errors: {self.errors}')
+        sys.exit(2)
+
+    def _parse_request(self, url, method='post', **kwargs):
+        if method not in ['post', 'get', 'patch']:
+            self._process_failure(f'Invalid method: {method}')
+        request_method = getattr(requests, method)
+        try:
+            response = request_method(url, **kwargs)
+        except requests.exceptions.ConnectionError:
+            print(f'Connection error raised when connecting to {url}') if self.verbose else None
+            self._process_failure(f'Error connecting to url: {url}')
+
+        if not response.ok:
+            print(f'Invalid response returned from {url}') if self.verbose else None
+            self.errors += response.json()['errors']
+            return None
+        return response.json()
+
+    def _retrieve_updated_attributes(self):
+        print(f'Updating attributes to remove update random and sequential variables') if self.verbose else None
+        updated_attributes = deepcopy(self.attributes)
+        for key, value in updated_attributes.items():
+            if value == "##RAND##":
+                updated_attributes[key] = str(uuid4())
+            elif value == "##NUM##":
+                updated_attributes[key] = self.sequence_number
+        if self.device:
+            device_response = self._parse_request(
+                f"{self.profiles_url}/api/v1/device",
+                data={'device_type': "AXLE_GATEWAY"},
+                headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+            if not device_response:
+                self._process_failure(f'Error generating device')
+            updated_attributes['device_id'] = device_response['data']['id']
+        self.sequence_number += 1
+        return updated_attributes
+
+    def _set_user_tokens(self):
+        for user in self.active_users:
+            if self.users[user]['token_exp'] and (self.users[user]['token_exp'] < datetime.now(timezone.utc)):
+                continue
+            response = self.get_user_jwt(self.users[user]['username'], self.users[user]['password'])
+            if not response:
+                self._process_failure(f'Error generating tokens')
+
+            self.users[user]['token'] = response['id_token']
+            self.users[user]['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=response['expires_in'])
+
+    def get_user_jwt(self, username, password):
+        response = self._parse_request(
+            f"{self.profiles_url}/openid/token/",
+            data={
+                "username": username,
+                "password": password,
+                "client_id": self.client_id,
+                "grant_type": "password",
+                "scope": "openid email"
+            })
+        if not response:
+            print(self.errors)
+            sys.exit(2)
+
+        return response
+
+    def _set_wallets(self):
+        print('Generating wallets for shipment') if self.verbose else None
+        shipper_response = self._parse_request(
+            f"{self.profiles_url}/api/v1/wallet/generate",
+            headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+        carrier_response = self._parse_request(
+            f"{self.profiles_url}/api/v1/wallet/generate",
+            headers={'Authorization': 'JWT {}'.format(self.carrier['token'])})
+        if not shipper_response or not carrier_response:
+            print(f'Error generating {"shipper" if not shipper_response else "carrier"} wallet.') \
+                if self.verbose else None
+
+            self._process_failure(f'Error generating wallets')
+
+        if self.moderator:
+            moderator_response = self._parse_request(
+                f"{self.profiles_url}/api/v1/wallet/generate/",
+                headers={'Authorization': 'JWT {}'.format(self.moderator['token'])})
+            if not moderator_response:
+                print(f'Error generating moderator wallet.') if self.verbose else None
+                self._process_failure(f'Error generating moderator wallets')
+
+            self.attributes['moderator_wallet_id'] = moderator_response['data']['id']
+        self.attributes['shipper_wallet_id'] = shipper_response['data']['id']
+        self.attributes['carrier_wallet_id'] = carrier_response['data']['id']
+
+    def _set_storage_credentials(self):
+        print(f'Creating storage credentials.') if self.verbose else None
+
+        response = self._parse_request(
+            f"{self.profiles_url}/api/v1/storage_credentials", data={
+                'driver_type': 'local',
+                'base_path': '/shipments',
+                'options': "{}",
+                'title': f'Shipment creator SC: {str(uuid4())}'
+            }, headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+
+        if not response:
+            self._process_failure(f'Error generating storage credentials')
+
+        print(f'Created storage credentials: {response["data"]["id"]}') if self.verbose else None
+        self.attributes['storage_credentials_id'] = response['data']['id']
+
+    def _create_shipment(self):
+        attributes = self._retrieve_updated_attributes()
+        response = self._parse_request(
+            f"{self.transmission_url}/api/v1/shipments/",
+            data=attributes,
+            headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+        if not response:
+            return
+        self.shipments.append(response['data']['id'])
+
+        if self.add_tracking:
+            print(f'Adding tracking via device: {attributes["device_id"]}') if self.verbose else None
+            self.tracking_data['device_id'] = response['data']['id']
+            tracking_data_collection = [{"payload": self.tracking_data} for i in range(10)]
+            tracking_response = requests.post(
+                f"{self.transmission_url}/api/v1/devices/{attributes['device_id']}/tracking",
+                json=tracking_data_collection,
+                headers={"Content-type": "application/json"}
+            )
+            if self.verbose:
+                print(f'{"Failed to add" if not tracking_response.ok else "Added"} tracking via device: {attributes["device_id"]}')
+
+        if self.add_telemetry:
+            device_id = attributes["device_id"]
+            print(f'Creating sensors for device: {device_id}') if self.verbose else None
+            telemetry_data = []
+            self.telemetry_data['device_id'] = device_id
+            for i in range(3):
+                attributes = {
+                    'name': f'Sensor: {str(uuid4())}',
+                    'hardware_id': f'Hardware_id: {str(uuid4())}',
+                    'units': 'c',
+                }
+                response = self._parse_request(
+                    f'{self.profiles_url}/api/v1/device/{device_id}/sensor/',
+                    data=attributes,
+                    headers={'Authorization': 'JWT {}'.format(self.shipper['token'])}
+                )
+                if not response:
+                    print(f'{"Failed to create" if not response.ok else "created"} sensor for device: {device_id}')
+                    continue
+
+                telemetry_copy = deepcopy(self.telemetry_data)
+                telemetry_copy['hardware_id'] = attributes['hardware_id']
+                telemetry_copy['sensor_id'] = response['data']['id']
+                telemetry_copy['value'] = randint(0, 100)
+                telemetry_data += [{"payload": telemetry_copy} for i in range(3)]
+            print(f'Adding telemetry via device: {device_id}') if self.verbose else None
+            telemetry_response = requests.post(
+                f"{self.transmission_url}/api/v1/devices/{device_id}/telemetry",
+                json=telemetry_data,
+                headers={"Content-type": "application/json"}
+            )
+            if self.verbose:
+                print(f'{"Failed to add" if not telemetry_response.ok else "Added"} tracking via device: {device_id}')
+
+    def create_bulk_shipments(self):
+        print('Generating user tokens') if self.verbose else None
+        self._set_user_tokens()
+        self._set_wallets()
+        self._set_storage_credentials()
+        for i in range(self.total):
+            if i % self.chunk_size != 0:
+                self._set_user_tokens()
+                self._set_wallets()
+            self._create_shipment()
+
+        print(f'Shipments created: {len(self.shipments)}')
+        if self.verbose:
+            print(f'Shipment ids: {self.shipments}')
+        print(f'Response Errors: {self.errors}')
+
+
+shipment_creator = CreateShipments()
+shipment_creator.create_bulk_shipments()

From 751fd29aa84dafd4dbf9e6b4ad3d325c87a2c9ac Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Thu, 4 Jun 2020 10:18:41 -0400
Subject: [PATCH 02/11] Pylint fixes

---
 bin/dev-tools/create_shipments.py | 156 +++++++++++++++++-------------
 1 file changed, 88 insertions(+), 68 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 833c74a2..3d4284a1 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -1,16 +1,17 @@
 #! /usr/bin/env python3
 
 import argparse
-import json
 import sys
-from datetime import datetime, timezone, timedelta
+import json
 from json.decoder import JSONDecodeError
-from random import randint
+from datetime import datetime, timezone, timedelta
 from uuid import uuid4
 
-import requests
 from copy import deepcopy
+from random import randint
+import requests
 
+# pylint:disable=invalid-name
 parser = argparse.ArgumentParser()
 parser.add_argument("--total", "-t", help="Total amount of shipments to be created, defaults to 10", type=int)
 parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes", type=int)
@@ -32,6 +33,7 @@
 argument_list = sys.argv[1:]
 
 
+# pylint:disable=too-many-instance-attributes
 class CreateShipments:
     # Default variables for local use
     profiles_url = 'http://localhost:9000'
@@ -67,18 +69,15 @@ class CreateShipments:
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
 
-
     # Values that can be set via command line
     attributes = {}
-    add_tracking = False
-    add_telemetry = False
 
     # Values that cannot be set via command line but dynamic within function
-    active_users = {'user_1'}
     errors = []
     shipments = []
 
     def __init__(self):
+        self.active_users = {'user_1'}
         args = parser.parse_args()
         self.carrier = self._validate_user(args.carrier) if args.carrier else self.users['user_1']
         self.shipper = self._validate_user(args.shipper) if args.shipper else self.users['user_1']
@@ -88,12 +87,12 @@ def __init__(self):
         self.verbose = args.verbose
         self.device = args.device
         self.chunk_size = int(args.partition) if args.partition else 10
-        if args.add_tracking and not self.device:
-            parser.error(f'--add_tracking requires --device or -d')
         self.add_tracking = args.add_tracking
-        if args.add_telemetry and not self.device:
-            parser.error(f'--add_telemetry requires --device or -d')
+        if self.add_tracking and not self.device:
+            parser.error(f'--add_tracking requires --device or -d')
         self.add_telemetry = args.add_telemetry
+        if self.add_telemetry and not self.device:
+            parser.error(f'--add_telemetry requires --device or -d')
         if args.attributes:
             self._validate_attributes(args.attributes)
         if self.users['user_1'] not in (self.moderator, self.carrier, self.shipper):
@@ -110,13 +109,15 @@ def _validate_user(self, user):
         return self.users[user]
 
     def _validate_attributes(self, attributes):
-        print(f'Validating attributes: {attributes}') if self.verbose else None
+        if self.verbose:
+            print(f'Validating attributes: {attributes}')
         for attribute in attributes:
             try:
                 attribute = json.loads(attribute)
             except JSONDecodeError:
-                print(f'Non json attribute recieved: {attribute}') if self.verbose else None
-            if type(attribute) == dict:
+                if self.verbose:
+                    print(f'Non json attribute recieved: {attribute}')
+            if isinstance(attribute, dict):
                 self.attributes.update(attribute)
             else:
                 try:
@@ -137,17 +138,20 @@ def _parse_request(self, url, method='post', **kwargs):
         try:
             response = request_method(url, **kwargs)
         except requests.exceptions.ConnectionError:
-            print(f'Connection error raised when connecting to {url}') if self.verbose else None
+            if self.verbose:
+                print(f'Connection error raised when connecting to {url}')
             self._process_failure(f'Error connecting to url: {url}')
 
         if not response.ok:
-            print(f'Invalid response returned from {url}') if self.verbose else None
+            if self.verbose:
+                print(f'Invalid response returned from {url}')
             self.errors += response.json()['errors']
             return None
         return response.json()
 
     def _retrieve_updated_attributes(self):
-        print(f'Updating attributes to remove update random and sequential variables') if self.verbose else None
+        if self.verbose:
+            print(f'Updating attributes to remove update random and sequential variables')
         updated_attributes = deepcopy(self.attributes)
         for key, value in updated_attributes.items():
             if value == "##RAND##":
@@ -193,7 +197,8 @@ def get_user_jwt(self, username, password):
         return response
 
     def _set_wallets(self):
-        print('Generating wallets for shipment') if self.verbose else None
+        if self.verbose:
+            print('Generating wallets for shipment')
         shipper_response = self._parse_request(
             f"{self.profiles_url}/api/v1/wallet/generate",
             headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
@@ -201,8 +206,8 @@ def _set_wallets(self):
             f"{self.profiles_url}/api/v1/wallet/generate",
             headers={'Authorization': 'JWT {}'.format(self.carrier['token'])})
         if not shipper_response or not carrier_response:
-            print(f'Error generating {"shipper" if not shipper_response else "carrier"} wallet.') \
-                if self.verbose else None
+            if self.verbose:
+                print(f'Error generating {"shipper" if not shipper_response else "carrier"} wallet.')
 
             self._process_failure(f'Error generating wallets')
 
@@ -211,7 +216,8 @@ def _set_wallets(self):
                 f"{self.profiles_url}/api/v1/wallet/generate/",
                 headers={'Authorization': 'JWT {}'.format(self.moderator['token'])})
             if not moderator_response:
-                print(f'Error generating moderator wallet.') if self.verbose else None
+                if self.verbose:
+                    print(f'Error generating moderator wallet.')
                 self._process_failure(f'Error generating moderator wallets')
 
             self.attributes['moderator_wallet_id'] = moderator_response['data']['id']
@@ -219,7 +225,8 @@ def _set_wallets(self):
         self.attributes['carrier_wallet_id'] = carrier_response['data']['id']
 
     def _set_storage_credentials(self):
-        print(f'Creating storage credentials.') if self.verbose else None
+        if self.verbose:
+            print(f'Creating storage credentials.')
 
         response = self._parse_request(
             f"{self.profiles_url}/api/v1/storage_credentials", data={
@@ -232,9 +239,62 @@ def _set_storage_credentials(self):
         if not response:
             self._process_failure(f'Error generating storage credentials')
 
-        print(f'Created storage credentials: {response["data"]["id"]}') if self.verbose else None
+        if self.verbose:
+            print(f'Created storage credentials: {response["data"]["id"]}')
         self.attributes['storage_credentials_id'] = response['data']['id']
 
+    # pylint:disable=unused-variable
+    def _add_telemetry(self, device_id):
+        if self.verbose:
+            print(f'Creating sensors for device: {device_id}')
+        telemetry_data = []
+        self.telemetry_data['device_id'] = device_id
+        for i in range(3):
+            attributes = {
+                'name': f'Sensor: {str(uuid4())}',
+                'hardware_id': f'Hardware_id: {str(uuid4())}',
+                'units': 'c',
+            }
+            response = self._parse_request(
+                f'{self.profiles_url}/api/v1/device/{device_id}/sensor/',
+                data=attributes,
+                headers={'Authorization': 'JWT {}'.format(self.shipper['token'])}
+            )
+            if not response:
+                print(f'{"Failed to create" if not response.ok else "created"} sensor for device: {device_id}')
+                continue
+
+            telemetry_copy = deepcopy(self.telemetry_data)
+            telemetry_copy['hardware_id'] = attributes['hardware_id']
+            telemetry_copy['sensor_id'] = response['data']['id']
+            for j in range(3):
+                telemetry_copy['value'] = randint(0, 100)
+                telemetry_data.append({"payload": telemetry_copy})
+        if self.verbose:
+            print(f'Adding telemetry via device: {device_id}')
+        telemetry_response = requests.post(
+            f"{self.transmission_url}/api/v1/devices/{device_id}/telemetry",
+            json=telemetry_data,
+            headers={"Content-type": "application/json"}
+        )
+        if self.verbose:
+            print(f'{"Failed to add" if not telemetry_response.ok else "Added"} tracking via device: {device_id}')
+
+    def _add_tracking(self, device_id):
+        if self.verbose:
+            print(f'Adding tracking via device: {device_id}')
+        self.tracking_data['device_id'] = device_id
+        # py
+        tracking_data_collection = [{"payload": self.tracking_data} for i in range(10)]
+        tracking_response = requests.post(
+            f"{self.transmission_url}/api/v1/devices/{device_id}/tracking",
+            json=tracking_data_collection,
+            headers={"Content-type": "application/json"}
+        )
+        if self.verbose:
+            print(f'{"Failed to add" if not tracking_response.ok else "Added"} '
+                  f'tracking via device: {device_id}')
+
     def _create_shipment(self):
         attributes = self._retrieve_updated_attributes()
         response = self._parse_request(
@@ -246,53 +306,13 @@ def _create_shipment(self):
         self.shipments.append(response['data']['id'])
 
         if self.add_tracking:
-            print(f'Adding tracking via device: {attributes["device_id"]}') if self.verbose else None
-            self.tracking_data['device_id'] = response['data']['id']
-            tracking_data_collection = [{"payload": self.tracking_data} for i in range(10)]
-            tracking_response = requests.post(
-                f"{self.transmission_url}/api/v1/devices/{attributes['device_id']}/tracking",
-                json=tracking_data_collection,
-                headers={"Content-type": "application/json"}
-            )
-            if self.verbose:
-                print(f'{"Failed to add" if not tracking_response.ok else "Added"} tracking via device: {attributes["device_id"]}')
-
+            self._add_tracking(attributes['device_id'])
         if self.add_telemetry:
-            device_id = attributes["device_id"]
-            print(f'Creating sensors for device: {device_id}') if self.verbose else None
-            telemetry_data = []
-            self.telemetry_data['device_id'] = device_id
-            for i in range(3):
-                attributes = {
-                    'name': f'Sensor: {str(uuid4())}',
-                    'hardware_id': f'Hardware_id: {str(uuid4())}',
-                    'units': 'c',
-                }
-                response = self._parse_request(
-                    f'{self.profiles_url}/api/v1/device/{device_id}/sensor/',
-                    data=attributes,
-                    headers={'Authorization': 'JWT {}'.format(self.shipper['token'])}
-                )
-                if not response:
-                    print(f'{"Failed to create" if not response.ok else "created"} sensor for device: {device_id}')
-                    continue
-
-                telemetry_copy = deepcopy(self.telemetry_data)
-                telemetry_copy['hardware_id'] = attributes['hardware_id']
-                telemetry_copy['sensor_id'] = response['data']['id']
-                telemetry_copy['value'] = randint(0, 100)
-                telemetry_data += [{"payload": telemetry_copy} for i in range(3)]
-            print(f'Adding telemetry via device: {device_id}') if self.verbose else None
-            telemetry_response = requests.post(
-                f"{self.transmission_url}/api/v1/devices/{device_id}/telemetry",
-                json=telemetry_data,
-                headers={"Content-type": "application/json"}
-            )
-            if self.verbose:
-                print(f'{"Failed to add" if not telemetry_response.ok else "Added"} tracking via device: {device_id}')
+            self._add_telemetry(attributes['device_id'])
 
     def create_bulk_shipments(self):
-        print('Generating user tokens') if self.verbose else None
+        if self.verbose:
+            print('Generating user tokens')
         self._set_user_tokens()
         self._set_wallets()
         self._set_storage_credentials()

From 1a97124e94f50859e21afea44c02273d464e76cd Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Fri, 12 Jun 2020 15:41:44 -0400
Subject: [PATCH 03/11] Remove verbose and use logging instead. Move parsing of
 args into separate function

---
 bin/dev-tools/create_shipments.py | 352 ++++++++++++++++--------------
 1 file changed, 194 insertions(+), 158 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 3d4284a1..12379fd4 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -6,46 +6,88 @@
 from json.decoder import JSONDecodeError
 from datetime import datetime, timezone, timedelta
 from uuid import uuid4
+from enum import Enum
+import logging
 
 from copy import deepcopy
 from random import randint
 import requests
 
 # pylint:disable=invalid-name
+logger = logging.getLogger('transmission')
+
+
+class CriticalError(Exception):
+    def __init__(self, message):
+        logger.critical(message)
+        self.parameter = message
+
+    def __str__(self):
+        return repr(self.parameter)
+
+
+class NonCriticalError(Exception):
+    def __init__(self, message):
+        logger.warning(message)
+        self.parameter = message
+
+    def __str__(self):
+        return repr(self.parameter)
+
+
+class LogLevels(Enum):
+    critical = 'CRITICAL'
+    error = 'ERROR'
+    warning = 'WARNING'
+    info = 'INFO'
+    debug = 'DEBUG'
+
+    def __str__(self):
+        return self.value
+
+
 parser = argparse.ArgumentParser()
-parser.add_argument("--total", "-t", help="Total amount of shipments to be created, defaults to 10", type=int)
-parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes", type=int)
-parser.add_argument("--partition", "-p", help="Number shipments to create per wallet, defaults to 10", type=int)
-parser.add_argument("--carrier", "-c", help="Set carrier wallet owner (defaults to user 1)")
-parser.add_argument("--shipper", "-s", help="Set shipper wallet owner (defaults to user 1)")
-parser.add_argument("--moderator", "-m", help="Set moderator wallet owner (defaults to None)")
-parser.add_argument("--verbose", "-v", help="More descriptive when running functions.", action="store_true")
+parser.add_argument("--total", "-t", help="Total amount of shipments to be created, defaults to 10", type=int,
+                    default=10)
+parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes", type=int,
+                    default=0)
+parser.add_argument("--partition", "-p", help="Number shipments to create per wallet, defaults to 10", type=int,
+                    default=10)
+parser.add_argument("--carrier", "-c", help="Set carrier wallet owner by passing in the username. Defaults to user1.",
+                    choices=['user1@shipchain.io', 'user2@shipchain.io'], default='user1@shipchain.io')
+parser.add_argument("--shipper", "-s", help="Set shipper wallet owner by passing in the username. Defaults to user1.",
+                    choices=['user1@shipchain.io', 'user2@shipchain.io'], default='user1@shipchain.io')
+parser.add_argument("--moderator", "-m",
+                    help="Set moderator wallet owner by passing in the username. Defaults to user1.",
+                    choices=['user1@shipchain.io', 'user2@shipchain.io'])
+parser.add_argument("--loglevel", "-l", help="Set the logging level for the creator. Defaults to warning.",
+                    choices=list(LogLevels.__members__), default='warning')
 parser.add_argument("--device", "-d", help="Add devices to shipment creation, defaults to false",
                     action="store_true")
 parser.add_argument("--attributes", "-a", action='append',
-                    help="Attributes to be sent in shipment creation (defaults to required values)")
+                    help="Attributes to be sent in shipment creation (defaults to required values). "
+                         "Format should be either {\"key\": \"value\"} or key = value.")
 parser.add_argument("--add_tracking", help="Adds tracking data to shipments (requires --device or -d)",
                     action='store_true')
 parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
                     action='store_true')
-
-# Keep all but the first command-line arguments
-argument_list = sys.argv[1:]
+parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to localhost:9000",
+                    default='http://localhost:9000')
+parser.add_argument("--transmission_url", help="Sets the transmission url for the creator. Defaults to localhost:8000",
+                    default='http://localhost:8000')
 
 
 # pylint:disable=too-many-instance-attributes
-class CreateShipments:
+class ShipmentCreator:
     # Default variables for local use
-    profiles_url = 'http://localhost:9000'
-    transmission_url = 'http://localhost:8000'
     client_id = 892633
     users = {
-        'user_1': {
+        'user1@shipchain.io': {
             'username': 'user1@shipchain.io',
             'password': 'user1Password',
             'token': None,
             'token_exp': None,
-        }, 'user_2': {
+        }, 'user2@shipchain.io': {
             'username': 'user2@shipchain.io',
             'password': 'user2Password',
             'token': None,
@@ -69,89 +111,72 @@ class CreateShipments:
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
 
-    # Values that can be set via command line
-    attributes = {}
-
-    # Values that cannot be set via command line but dynamic within function
-    errors = []
-    shipments = []
-
     def __init__(self):
-        self.active_users = {'user_1'}
-        args = parser.parse_args()
-        self.carrier = self._validate_user(args.carrier) if args.carrier else self.users['user_1']
-        self.shipper = self._validate_user(args.shipper) if args.shipper else self.users['user_1']
-        self.moderator = self._validate_user(args.moderator) if args.moderator else None
-        self.sequence_number = args.startnumber if args.startnumber else 0
-        self.total = int(args.total) if args.total else 10
-        self.verbose = args.verbose
-        self.device = args.device
-        self.chunk_size = int(args.partition) if args.partition else 10
-        self.add_tracking = args.add_tracking
+        self.attributes = {}
+        self.errors = []
+        self.shipments = []
+
+    # pylint: disable=attribute-defined-outside-init
+    def handle_args(self, command_args):
+        self.carrier = self.users[command_args.carrier]
+        self.shipper = self.users[command_args.shipper]
+        self.moderator = self.users[command_args.moderator] if command_args.moderator else None
+        self.sequence_number = command_args.startnumber
+        self.total = int(command_args.total)
+        console = logging.StreamHandler()
+        console.setLevel(LogLevels[command_args.loglevel].value)
+        logger.addHandler(console)
+        logger.setLevel(LogLevels[command_args.loglevel].value)
+        self.device = command_args.device
+        self.chunk_size = int(command_args.partition)
+        self.add_tracking = command_args.add_tracking
         if self.add_tracking and not self.device:
             parser.error(f'--add_tracking requires --device or -d')
-        self.add_telemetry = args.add_telemetry
+        self.add_telemetry = command_args.add_telemetry
+        self.profiles_url = command_args.profiles_url
+        self.transmission_url = command_args.transmission_url
         if self.add_telemetry and not self.device:
             parser.error(f'--add_telemetry requires --device or -d')
-        if args.attributes:
-            self._validate_attributes(args.attributes)
-        if self.users['user_1'] not in (self.moderator, self.carrier, self.shipper):
-            self.active_users.remove('user_1')
+        if command_args.attributes:
+            self._validate_attributes(command_args.attributes)
 
     def _chunker(self, array):
         return (array[i:i + self.chunk_size] for i in range(0, len(array), self.chunk_size))
 
-    def _validate_user(self, user):
-        if user not in self.users:
-            self._process_failure(f'Invalid user: {user}')
-        if user not in self.active_users:
-            self.active_users.add(user)
-        return self.users[user]
-
     def _validate_attributes(self, attributes):
-        if self.verbose:
-            print(f'Validating attributes: {attributes}')
+        logger.info(f'Validating attributes: {attributes}')
         for attribute in attributes:
             try:
                 attribute = json.loads(attribute)
             except JSONDecodeError:
-                if self.verbose:
-                    print(f'Non json attribute recieved: {attribute}')
+                logger.info(f'Non json attribute recieved: {attribute}')
             if isinstance(attribute, dict):
+                logger.info(f'Dict attribute recieved: {attribute}')
                 self.attributes.update(attribute)
             else:
                 try:
                     key, value = attribute.split("=")
                 except ValueError:
-                    self._process_failure(f'Invalid format for attribute: {attribute}, should be in format: key=value')
+                    raise CriticalError(f'Invalid format for attribute: {attribute}, should be in format: key=value')
                 self.attributes[key] = value
 
-    def _process_failure(self, error_message):
-        print(f'Shipment Creator failed: {error_message}')
-        print(f'Response errors: {self.errors}')
-        sys.exit(2)
-
     def _parse_request(self, url, method='post', **kwargs):
         if method not in ['post', 'get', 'patch']:
-            self._process_failure(f'Invalid method: {method}')
+            raise CriticalError(f'Invalid method: {method}')
         request_method = getattr(requests, method)
         try:
             response = request_method(url, **kwargs)
         except requests.exceptions.ConnectionError:
-            if self.verbose:
-                print(f'Connection error raised when connecting to {url}')
-            self._process_failure(f'Error connecting to url: {url}')
+            raise CriticalError(f'Connection error raised when connecting to {url}')
 
         if not response.ok:
-            if self.verbose:
-                print(f'Invalid response returned from {url}')
             self.errors += response.json()['errors']
-            return None
+            raise NonCriticalError(f'Invalid response returned from {url}')
+
         return response.json()
 
     def _retrieve_updated_attributes(self):
-        if self.verbose:
-            print(f'Updating attributes to remove update random and sequential variables')
+        logger.info('Updating attributes to remove/update random and sequential variables')
         updated_attributes = deepcopy(self.attributes)
         for key, value in updated_attributes.items():
             if value == "##RAND##":
@@ -159,94 +184,88 @@ def _retrieve_updated_attributes(self):
             elif value == "##NUM##":
                 updated_attributes[key] = self.sequence_number
         if self.device:
-            device_response = self._parse_request(
-                f"{self.profiles_url}/api/v1/device",
-                data={'device_type': "AXLE_GATEWAY"},
-                headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
-            if not device_response:
-                self._process_failure(f'Error generating device')
+            try:
+                device_response = self._parse_request(
+                    f"{self.profiles_url}/api/v1/device",
+                    data={'device_type': "AXLE_GATEWAY"},
+                    headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+            except NonCriticalError:
+                raise CriticalError(f'Error generating device')
             updated_attributes['device_id'] = device_response['data']['id']
         self.sequence_number += 1
         return updated_attributes
 
-    def _set_user_tokens(self):
-        for user in self.active_users:
-            if self.users[user]['token_exp'] and (self.users[user]['token_exp'] < datetime.now(timezone.utc)):
-                continue
-            response = self.get_user_jwt(self.users[user]['username'], self.users[user]['password'])
-            if not response:
-                self._process_failure(f'Error generating tokens')
-
-            self.users[user]['token'] = response['id_token']
-            self.users[user]['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=response['expires_in'])
+    def get_user_jwt(self, user):
+        if user['token_exp'] and user['token_exp'] < datetime.now(timezone.utc):
+            logger.debug(f'User {user["username"]} has non-expired token')
+            return user['token']
 
-    def get_user_jwt(self, username, password):
         response = self._parse_request(
             f"{self.profiles_url}/openid/token/",
             data={
-                "username": username,
-                "password": password,
+                "username": user['username'],
+                "password": user['password'],
                 "client_id": self.client_id,
                 "grant_type": "password",
                 "scope": "openid email"
             })
         if not response:
-            print(self.errors)
-            sys.exit(2)
+            raise CriticalError(f'Error generating token for user {user["username"]}')
 
-        return response
+        user['token'] = response['id_token']
+        # Give a buffer of 15 seconds to the token time check
+        user['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=(response['expires_in'] - 15))
+        return response['id_token']
 
     def _set_wallets(self):
-        if self.verbose:
-            print('Generating wallets for shipment')
-        shipper_response = self._parse_request(
-            f"{self.profiles_url}/api/v1/wallet/generate",
-            headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
-        carrier_response = self._parse_request(
-            f"{self.profiles_url}/api/v1/wallet/generate",
-            headers={'Authorization': 'JWT {}'.format(self.carrier['token'])})
-        if not shipper_response or not carrier_response:
-            if self.verbose:
-                print(f'Error generating {"shipper" if not shipper_response else "carrier"} wallet.')
-
-            self._process_failure(f'Error generating wallets')
+        logger.info('Generating wallets for shipment')
+        try:
+            shipper_response = self._parse_request(
+                f"{self.profiles_url}/api/v1/wallet/generate",
+                headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
+        except NonCriticalError:
+            raise CriticalError(f'Error generating shipper wallet for user {self.shipper["username"]}.')
+        try:
+            carrier_response = self._parse_request(
+                f"{self.profiles_url}/api/v1/wallet/generate",
+                headers={'Authorization': f'JWT {self.get_user_jwt(self.carrier)}'})
+        except NonCriticalError:
+            raise CriticalError(f'Error generating carrier wallet for user {self.carrier["username"]}.')
 
         if self.moderator:
-            moderator_response = self._parse_request(
-                f"{self.profiles_url}/api/v1/wallet/generate/",
-                headers={'Authorization': 'JWT {}'.format(self.moderator['token'])})
-            if not moderator_response:
-                if self.verbose:
-                    print(f'Error generating moderator wallet.')
-                self._process_failure(f'Error generating moderator wallets')
+            try:
+                moderator_response = self._parse_request(
+                    f"{self.profiles_url}/api/v1/wallet/generate/",
+                    headers={'Authorization': f'JWT {self.get_user_jwt(self.moderator)}'})
+            except NonCriticalError:
+                raise CriticalError(f'Error generating storage credentials for user {self.shipper["username"]}.')
 
             self.attributes['moderator_wallet_id'] = moderator_response['data']['id']
         self.attributes['shipper_wallet_id'] = shipper_response['data']['id']
         self.attributes['carrier_wallet_id'] = carrier_response['data']['id']
 
     def _set_storage_credentials(self):
-        if self.verbose:
-            print(f'Creating storage credentials.')
-
-        response = self._parse_request(
-            f"{self.profiles_url}/api/v1/storage_credentials", data={
-                'driver_type': 'local',
-                'base_path': '/shipments',
-                'options': "{}",
-                'title': f'Shipment creator SC: {str(uuid4())}'
-            }, headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
+        logger.info('Creating storage credentials.')
+        try:
+            response = self._parse_request(
+                f"{self.profiles_url}/api/v1/storage_credentials", data={
+                    'driver_type': 'local',
+                    'base_path': '/shipments',
+                    'options': "{}",
+                    'title': f'Shipment creator SC: {str(uuid4())}'
+                }, headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
+        except NonCriticalError:
+            raise CriticalError(f'Error generating storage .')
 
         if not response:
-            self._process_failure(f'Error generating storage credentials')
+            raise CriticalError(f'Error generating storage credentials.')
 
-        if self.verbose:
-            print(f'Created storage credentials: {response["data"]["id"]}')
+        logger.debug(f'Created storage credentials: {response["data"]["id"]}')
         self.attributes['storage_credentials_id'] = response['data']['id']
 
     # pylint:disable=unused-variable
     def _add_telemetry(self, device_id):
-        if self.verbose:
-            print(f'Creating sensors for device: {device_id}')
+        logger.info(f'Creating sensors for device: {device_id}')
         telemetry_data = []
         self.telemetry_data['device_id'] = device_id
         for i in range(3):
@@ -255,54 +274,62 @@ def _add_telemetry(self, device_id):
                 'hardware_id': f'Hardware_id: {str(uuid4())}',
                 'units': 'c',
             }
-            response = self._parse_request(
-                f'{self.profiles_url}/api/v1/device/{device_id}/sensor/',
-                data=attributes,
-                headers={'Authorization': 'JWT {}'.format(self.shipper['token'])}
-            )
-            if not response:
-                print(f'{"Failed to create" if not response.ok else "created"} sensor for device: {device_id}')
+            try:
+                response = self._parse_request(
+                    f'{self.profiles_url}/api/v1/device/{device_id}/sensor/',
+                    data=attributes,
+                    headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'}
+                )
+
+            except NonCriticalError:
+                logger.warning(f'Failed to create sensor for device: {device_id}')
                 continue
 
+            logger.info(f'Created sensor {response["data"]["id"]} for device {device_id}')
+
             telemetry_copy = deepcopy(self.telemetry_data)
             telemetry_copy['hardware_id'] = attributes['hardware_id']
             telemetry_copy['sensor_id'] = response['data']['id']
             for j in range(3):
                 telemetry_copy['value'] = randint(0, 100)
                 telemetry_data.append({"payload": telemetry_copy})
-        if self.verbose:
-            print(f'Adding telemetry via device: {device_id}')
+
+        logger.info(f'Adding telemetry via device: {device_id}')
         telemetry_response = requests.post(
             f"{self.transmission_url}/api/v1/devices/{device_id}/telemetry",
             json=telemetry_data,
             headers={"Content-type": "application/json"}
         )
-        if self.verbose:
-            print(f'{"Failed to add" if not telemetry_response.ok else "Added"} tracking via device: {device_id}')
+        if not telemetry_response.ok:
+            logger.warning(f'Failed to add telemetry data for device: {device_id}')
+        else:
+            logger.info(f'Added telemetry data via device: {device_id}')
 
     def _add_tracking(self, device_id):
-        if self.verbose:
-            print(f'Adding tracking via device: {device_id}')
+        logger.info(f'Adding tracking via device: {device_id}')
         self.tracking_data['device_id'] = device_id
-        # py
         tracking_data_collection = [{"payload": self.tracking_data} for i in range(10)]
         tracking_response = requests.post(
             f"{self.transmission_url}/api/v1/devices/{device_id}/tracking",
             json=tracking_data_collection,
             headers={"Content-type": "application/json"}
         )
-        if self.verbose:
-            print(f'{"Failed to add" if not tracking_response.ok else "Added"} '
-                  f'tracking via device: {device_id}')
+        if not tracking_response.ok:
+            logger.warning(f'Failed to add tracking data for device: {device_id}')
+        else:
+            logger.info(f'Added tracking data via device: {device_id}')
 
     def _create_shipment(self):
         attributes = self._retrieve_updated_attributes()
-        response = self._parse_request(
-            f"{self.transmission_url}/api/v1/shipments/",
-            data=attributes,
-            headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
-        if not response:
+        try:
+            response = self._parse_request(
+                f"{self.transmission_url}/api/v1/shipments/",
+                data=attributes,
+                headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
+        except NonCriticalError:
+            logger.warning(f'Failed to create shipment.')
             return
+
         self.shipments.append(response['data']['id'])
 
         if self.add_tracking:
@@ -311,22 +338,31 @@ def _create_shipment(self):
             self._add_telemetry(attributes['device_id'])
 
     def create_bulk_shipments(self):
-        if self.verbose:
-            print('Generating user tokens')
-        self._set_user_tokens()
-        self._set_wallets()
-        self._set_storage_credentials()
+        try:
+            self._set_wallets()
+            self._set_storage_credentials()
+        except CriticalError:
+            logger.critical(f'Response errors: {self.errors}')
+            sys.exit(2)
+
         for i in range(self.total):
             if i % self.chunk_size != 0:
-                self._set_user_tokens()
-                self._set_wallets()
-            self._create_shipment()
+                try:
+                    self._set_wallets()
+                except CriticalError:
+                    logger.critical(f'Response errors: {self.errors}')
+                    break
+            try:
+                self._create_shipment()
+            except NonCriticalError:
+                logger.info(f'Errors when generating shipments: {self.errors}')
 
-        print(f'Shipments created: {len(self.shipments)}')
-        if self.verbose:
-            print(f'Shipment ids: {self.shipments}')
-        print(f'Response Errors: {self.errors}')
+        logger.info(f'Shipments created: {len(self.shipments)}')
+        logger.info(f'Shipment ids: {self.shipments}')
 
 
-shipment_creator = CreateShipments()
-shipment_creator.create_bulk_shipments()
+if __name__ == '__main__':
+    args = parser.parse_args()
+    shipment_creator = ShipmentCreator()
+    shipment_creator.handle_args(args)
+    shipment_creator.create_bulk_shipments()

From 8118e42f8e449b242970dbe7c1ff64271bda24e0 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Fri, 12 Jun 2020 15:47:54 -0400
Subject: [PATCH 04/11] Set buffer time for token check to 60 seconds from 15
 seconds

---
 bin/dev-tools/create_shipments.py | 35 ++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 12379fd4..804b6cf3 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -111,12 +111,31 @@ class ShipmentCreator:
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
 
-    def __init__(self):
+    # pylint: disable=too-many-arguments, too-many-locals
+    def __init__(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=None, sequence_number=0,
+                 total=10, loglevel='warning', device=False, partition=10, attributes=None, add_tracking=False,
+                 add_telemetry=False, profiles_url='http://localhost:9000', transmission_url='http://localhost:8000'):
         self.attributes = {}
         self.errors = []
         self.shipments = []
+        self.carrier = self.users[carrier]
+        self.shipper = self.users[shipper]
+        self.moderator = self.users[moderator] if moderator else None
+        self.sequence_number = sequence_number
+        self.total = total
+        console = logging.StreamHandler()
+        console.setLevel(LogLevels[loglevel].value)
+        logger.addHandler(console)
+        logger.setLevel(LogLevels[loglevel].value)
+        self.device = device
+        self.chunk_size = partition
+        self.add_tracking = add_tracking
+        self.add_telemetry = add_telemetry
+        self.profiles_url = profiles_url
+        self.transmission_url = transmission_url
+        if attributes:
+            self._validate_attributes(attributes)
 
-    # pylint: disable=attribute-defined-outside-init
     def handle_args(self, command_args):
         self.carrier = self.users[command_args.carrier]
         self.shipper = self.users[command_args.shipper]
@@ -133,10 +152,10 @@ def handle_args(self, command_args):
         if self.add_tracking and not self.device:
             parser.error(f'--add_tracking requires --device or -d')
         self.add_telemetry = command_args.add_telemetry
-        self.profiles_url = command_args.profiles_url
-        self.transmission_url = command_args.transmission_url
         if self.add_telemetry and not self.device:
             parser.error(f'--add_telemetry requires --device or -d')
+        self.profiles_url = command_args.profiles_url
+        self.transmission_url = command_args.transmission_url
         if command_args.attributes:
             self._validate_attributes(command_args.attributes)
 
@@ -149,9 +168,9 @@ def _validate_attributes(self, attributes):
             try:
                 attribute = json.loads(attribute)
             except JSONDecodeError:
-                logger.info(f'Non json attribute recieved: {attribute}')
+                logger.debug(f'Non json attribute recieved: {attribute}')
             if isinstance(attribute, dict):
-                logger.info(f'Dict attribute recieved: {attribute}')
+                logger.debug(f'Dict attribute recieved: {attribute}')
                 self.attributes.update(attribute)
             else:
                 try:
@@ -213,8 +232,8 @@ def get_user_jwt(self, user):
             raise CriticalError(f'Error generating token for user {user["username"]}')
 
         user['token'] = response['id_token']
-        # Give a buffer of 15 seconds to the token time check
-        user['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=(response['expires_in'] - 15))
+        # Give a buffer of 60 seconds to the token time check
+        user['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=(response['expires_in'] - 60))
         return response['id_token']
 
     def _set_wallets(self):

From cfebe9a6aaf766e321ccbe853580c441ba8ff492 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Tue, 16 Jun 2020 14:14:34 -0400
Subject: [PATCH 05/11] Remove duplicate error response and add validation
 check to urls

---
 bin/dev-tools/create_shipments.py | 63 +++++++++++++++----------------
 1 file changed, 30 insertions(+), 33 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 804b6cf3..f948e847 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -5,6 +5,7 @@
 import json
 from json.decoder import JSONDecodeError
 from datetime import datetime, timezone, timedelta
+from urllib.parse import urlparse
 from uuid import uuid4
 from enum import Enum
 import logging
@@ -15,11 +16,13 @@
 
 # pylint:disable=invalid-name
 logger = logging.getLogger('transmission')
+console = logging.StreamHandler()
 
 
 class CriticalError(Exception):
     def __init__(self, message):
         logger.critical(message)
+        logger.removeHandler(console)
         self.parameter = message
 
     def __str__(self):
@@ -71,9 +74,10 @@ def __str__(self):
                     action='store_true')
 parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
                     action='store_true')
-parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to localhost:9000",
+parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to http://localhost:9000",
                     default='http://localhost:9000')
-parser.add_argument("--transmission_url", help="Sets the transmission url for the creator. Defaults to localhost:8000",
+parser.add_argument("--transmission_url",
+                    help="Sets the transmission url for the creator. Defaults to http://localhost:8000",
                     default='http://localhost:8000')
 
 
@@ -112,18 +116,21 @@ class ShipmentCreator:
     }
 
     # pylint: disable=too-many-arguments, too-many-locals
-    def __init__(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=None, sequence_number=0,
-                 total=10, loglevel='warning', device=False, partition=10, attributes=None, add_tracking=False,
-                 add_telemetry=False, profiles_url='http://localhost:9000', transmission_url='http://localhost:8000'):
+    def __init__(self):
         self.attributes = {}
         self.errors = []
         self.shipments = []
+
+    # pylint: disable=too-many-arguments, too-many-locals, attribute-defined-outside-init
+    def set_shipment_fields(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=None,
+                            startnumber=0, total=10, loglevel='warning', device=False, partition=10, attributes=None,
+                            add_tracking=False, add_telemetry=False, profiles_url='http://localhost:9000',
+                            transmission_url='http://localhost:8000'):
         self.carrier = self.users[carrier]
         self.shipper = self.users[shipper]
         self.moderator = self.users[moderator] if moderator else None
-        self.sequence_number = sequence_number
+        self.sequence_number = startnumber
         self.total = total
-        console = logging.StreamHandler()
         console.setLevel(LogLevels[loglevel].value)
         logger.addHandler(console)
         logger.setLevel(LogLevels[loglevel].value)
@@ -131,37 +138,23 @@ def __init__(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', m
         self.chunk_size = partition
         self.add_tracking = add_tracking
         self.add_telemetry = add_telemetry
-        self.profiles_url = profiles_url
-        self.transmission_url = transmission_url
+        self.profiles_url = self._validate_url(profiles_url, 'profiles')
+        self.transmission_url = self._validate_url(transmission_url, 'transmission')
         if attributes:
             self._validate_attributes(attributes)
 
-    def handle_args(self, command_args):
-        self.carrier = self.users[command_args.carrier]
-        self.shipper = self.users[command_args.shipper]
-        self.moderator = self.users[command_args.moderator] if command_args.moderator else None
-        self.sequence_number = command_args.startnumber
-        self.total = int(command_args.total)
-        console = logging.StreamHandler()
-        console.setLevel(LogLevels[command_args.loglevel].value)
-        logger.addHandler(console)
-        logger.setLevel(LogLevels[command_args.loglevel].value)
-        self.device = command_args.device
-        self.chunk_size = int(command_args.partition)
-        self.add_tracking = command_args.add_tracking
-        if self.add_tracking and not self.device:
-            parser.error(f'--add_tracking requires --device or -d')
-        self.add_telemetry = command_args.add_telemetry
-        if self.add_telemetry and not self.device:
-            parser.error(f'--add_telemetry requires --device or -d')
-        self.profiles_url = command_args.profiles_url
-        self.transmission_url = command_args.transmission_url
-        if command_args.attributes:
-            self._validate_attributes(command_args.attributes)
-
     def _chunker(self, array):
         return (array[i:i + self.chunk_size] for i in range(0, len(array), self.chunk_size))
 
+    def _validate_url(self, url, url_base):
+        try:
+            result = urlparse(url)
+            if not result.scheme or not result.netloc:
+                raise CriticalError(f'Invalid url supplied for {url_base}: {url}')
+            return url
+        except ValueError:
+            raise CriticalError(f'Invalid url supplied for {url_base}: {url}')
+
     def _validate_attributes(self, attributes):
         logger.info(f'Validating attributes: {attributes}')
         for attribute in attributes:
@@ -382,6 +375,10 @@ def create_bulk_shipments(self):
 
 if __name__ == '__main__':
     args = parser.parse_args()
+    args_dict = vars(args)
     shipment_creator = ShipmentCreator()
-    shipment_creator.handle_args(args)
+    try:
+        shipment_creator.set_shipment_fields(**args_dict)
+    except CriticalError:
+        sys.exit(2)
     shipment_creator.create_bulk_shipments()

From 3bec96eae23bb739a9563dcbd8bf669ea82d2214 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Mon, 13 Jul 2020 12:54:07 -0400
Subject: [PATCH 06/11] Add ability to reuse wallets/SC and change total to
 count

---
 bin/dev-tools/create_shipments.py | 110 +++++++++++++++++++++++-------
 1 file changed, 85 insertions(+), 25 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index f948e847..477226b6 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -50,12 +50,12 @@ def __str__(self):
 
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--total", "-t", help="Total amount of shipments to be created, defaults to 10", type=int,
+parser.add_argument("--count", "-t", help="Total amount of shipments to be created, defaults to 10", type=int,
                     default=10)
-parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes", type=int,
+parser.add_argument("--startnumber", "-n", help="Number to start at for sequential attributes.", type=int,
                     default=0)
-parser.add_argument("--partition", "-p", help="Number shipments to create per wallet, defaults to 10", type=int,
-                    default=10)
+parser.add_argument("--partition", "-p", help="Maximum number of shipments to be created per wallet, defaults to 10.",
+                    type=int, default=10)
 parser.add_argument("--carrier", "-c", help="Set carrier wallet owner by passing in the username. Defaults to user1.",
                     choices=['user1@shipchain.io', 'user2@shipchain.io'], default='user1@shipchain.io')
 parser.add_argument("--shipper", "-s", help="Set shipper wallet owner by passing in the username. Defaults to user1.",
@@ -63,13 +63,16 @@ def __str__(self):
 parser.add_argument("--moderator", "-m",
                     help="Set moderator wallet owner by passing in the username. Defaults to user1.",
                     choices=['user1@shipchain.io', 'user2@shipchain.io'])
-parser.add_argument("--loglevel", "-l", help="Set the logging level for the creator. Defaults to warning.",
-                    choices=list(LogLevels.__members__), default='warning')
+parser.add_argument("--loglevel", "-l", help="Set the logging level for the creator. Defaults to info.",
+                    choices=list(LogLevels.__members__), default='info')
 parser.add_argument("--device", "-d", help="Add devices to shipment creation, defaults to false",
                     action="store_true")
 parser.add_argument("--attributes", "-a", action='append',
                     help="Attributes to be sent in shipment creation (defaults to required values). "
-                         "Format should be either {\"key\": \"value\"} or key = value.")
+                         "Setting a key to ##RAND## sets the value as new uuid per creation, "
+                         "and ##NUMB## is replaced with a number starting from the --startnumber "
+                         "Format should be either {\"key\": \"value\"} or key = value. "
+                         "EX: \'{\"forwarders_shipper_id\": \"##RAND##\", \"pro_number\": \"PRO_##NUMB##\"}\'")
 parser.add_argument("--add_tracking", help="Adds tracking data to shipments (requires --device or -d)",
                     action='store_true')
 parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
@@ -79,6 +82,13 @@ def __str__(self):
 parser.add_argument("--transmission_url",
                     help="Sets the transmission url for the creator. Defaults to http://localhost:8000",
                     default='http://localhost:8000')
+parser.add_argument("--reuse_wallets", help="Reuse wallets instead of creating new ones, defaults to True",
+                    action="store_true",
+                    default=True)
+parser.add_argument("--reuse_storage_credentials",
+                    help="Reuse storage credentials instead of creating a new one, defaults to True",
+                    action="store_true",
+                    default=True)
 
 
 # pylint:disable=too-many-instance-attributes
@@ -91,11 +101,13 @@ class ShipmentCreator:
             'password': 'user1Password',
             'token': None,
             'token_exp': None,
+            'wallets': []
         }, 'user2@shipchain.io': {
             'username': 'user2@shipchain.io',
             'password': 'user2Password',
             'token': None,
             'token_exp': None,
+            'wallets': []
         }
     }
     tracking_data = {
@@ -122,10 +134,11 @@ def __init__(self):
         self.shipments = []
 
     # pylint: disable=too-many-arguments, too-many-locals, attribute-defined-outside-init
-    def set_shipment_fields(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=None,
+    def set_shipment_fields(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=False,
                             startnumber=0, total=10, loglevel='warning', device=False, partition=10, attributes=None,
                             add_tracking=False, add_telemetry=False, profiles_url='http://localhost:9000',
-                            transmission_url='http://localhost:8000'):
+                            transmission_url='http://localhost:8000', reuse_wallets=False,
+                            reuse_storage_credentials=False):
         self.carrier = self.users[carrier]
         self.shipper = self.users[shipper]
         self.moderator = self.users[moderator] if moderator else None
@@ -135,16 +148,18 @@ def set_shipment_fields(self, carrier='user1@shipchain.io', shipper='user1@shipc
         logger.addHandler(console)
         logger.setLevel(LogLevels[loglevel].value)
         self.device = device
-        self.chunk_size = partition
+        self.partition = partition
         self.add_tracking = add_tracking
         self.add_telemetry = add_telemetry
+        self.reuse_storage_credentials = reuse_storage_credentials
+        self.reuse_wallets = reuse_wallets
         self.profiles_url = self._validate_url(profiles_url, 'profiles')
         self.transmission_url = self._validate_url(transmission_url, 'transmission')
         if attributes:
             self._validate_attributes(attributes)
 
     def _chunker(self, array):
-        return (array[i:i + self.chunk_size] for i in range(0, len(array), self.chunk_size))
+        return (array[i:i + self.partition] for i in range(0, len(array), self.partition))
 
     def _validate_url(self, url, url_base):
         try:
@@ -179,9 +194,13 @@ def _parse_request(self, url, method='post', **kwargs):
         try:
             response = request_method(url, **kwargs)
         except requests.exceptions.ConnectionError:
-            raise CriticalError(f'Connection error raised when connecting to {url}')
+            url_group = 'Profiles' if self.profiles_url in url else 'Transmission'
+            raise CriticalError(
+                f'Connection error raised when connecting to {url}. Ensure service {url_group} is running.')
 
         if not response.ok:
+            if response.status_code == 503:
+                logger.warning('Ensure Engine services are running.')
             self.errors += response.json()['errors']
             raise NonCriticalError(f'Invalid response returned from {url}')
 
@@ -191,10 +210,10 @@ def _retrieve_updated_attributes(self):
         logger.info('Updating attributes to remove/update random and sequential variables')
         updated_attributes = deepcopy(self.attributes)
         for key, value in updated_attributes.items():
-            if value == "##RAND##":
-                updated_attributes[key] = str(uuid4())
-            elif value == "##NUM##":
-                updated_attributes[key] = self.sequence_number
+            if "##RAND##" in value:
+                updated_attributes[key] = value.replace("##RAND##", str(uuid4()))
+            elif "##NUM##" in value:
+                updated_attributes[key] = value.replace("##NUM##", self.sequence_number)
         if self.device:
             try:
                 device_response = self._parse_request(
@@ -229,7 +248,34 @@ def get_user_jwt(self, user):
         user['token_exp'] = datetime.now(timezone.utc) + timedelta(seconds=(response['expires_in'] - 60))
         return response['id_token']
 
-    def _set_wallets(self):
+    def _reuse_wallets(self):
+        logger.info('Reusing wallets for shipment')
+        for wallet_owner in ('shipper', 'carrier', 'moderator'):
+            wallet_owner_dict = getattr(self, wallet_owner)
+            if not wallet_owner_dict:
+                continue
+
+            if not wallet_owner_dict['wallets']:
+                logger.debug(f'Wallets not found for owner: {wallet_owner_dict["username"]}')
+                try:
+                    response = self._parse_request(
+                        f"{self.profiles_url}/api/v1/wallet?page_size=9999", method='get',
+                        headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
+                except NonCriticalError:
+                    raise CriticalError(f'Error retrieving wallets for user {wallet_owner_dict["username"]}.')
+
+                logger.debug(f"Wallet count returned: {response['meta']['pagination']['count']}")
+                if response['meta']['pagination']['count'] == 0 or response['meta']['pagination']['count'] \
+                        < self.total // self.partition:
+                    raise CriticalError(f'Not enough wallets for user {wallet_owner_dict["username"]}.')
+
+                wallet_owner_dict['wallets'] = response['data']
+
+            wallet = wallet_owner_dict['wallets'].pop()
+
+            self.attributes[f'{wallet_owner}_wallet_id'] = wallet['id']
+
+    def _generate_wallets(self):
         logger.info('Generating wallets for shipment')
         try:
             shipper_response = self._parse_request(
@@ -256,7 +302,22 @@ def _set_wallets(self):
         self.attributes['shipper_wallet_id'] = shipper_response['data']['id']
         self.attributes['carrier_wallet_id'] = carrier_response['data']['id']
 
-    def _set_storage_credentials(self):
+    def _reuse_storage_credentials(self):
+        logger.info('Reusing storage credentials.')
+        try:
+            response = self._parse_request(
+                f"{self.profiles_url}/api/v1/storage_credentials",
+                headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'}, method='get')
+        except NonCriticalError:
+            raise CriticalError(f'Error retrieving storage credentials')
+
+        logger.debug(f'Storage credentials count returned: {response["meta"]["pagination"]["count"]}')
+        if response['meta']['pagination']['count'] == 0:
+            raise CriticalError(f'No storage credentials found associated with account {self.carrier["username"]}')
+
+        self.attributes['storage_credentials_id'] = response['data'][0]['id']
+
+    def _generate_storage_credentials(self):
         logger.info('Creating storage credentials.')
         try:
             response = self._parse_request(
@@ -267,9 +328,6 @@ def _set_storage_credentials(self):
                     'title': f'Shipment creator SC: {str(uuid4())}'
                 }, headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
         except NonCriticalError:
-            raise CriticalError(f'Error generating storage .')
-
-        if not response:
             raise CriticalError(f'Error generating storage credentials.')
 
         logger.debug(f'Created storage credentials: {response["data"]["id"]}')
@@ -333,6 +391,7 @@ def _add_tracking(self, device_id):
 
     def _create_shipment(self):
         attributes = self._retrieve_updated_attributes()
+        logger.info('Creating shipment')
         try:
             response = self._parse_request(
                 f"{self.transmission_url}/api/v1/shipments/",
@@ -342,6 +401,7 @@ def _create_shipment(self):
             logger.warning(f'Failed to create shipment.')
             return
 
+        logger.debug(f'Created shipment: {response["data"]["id"]}')
         self.shipments.append(response['data']['id'])
 
         if self.add_tracking:
@@ -351,16 +411,16 @@ def _create_shipment(self):
 
     def create_bulk_shipments(self):
         try:
-            self._set_wallets()
-            self._set_storage_credentials()
+            (self._reuse_storage_credentials() if self.reuse_storage_credentials
+             else self._generate_storage_credentials())
         except CriticalError:
             logger.critical(f'Response errors: {self.errors}')
             sys.exit(2)
 
         for i in range(self.total):
-            if i % self.chunk_size != 0:
+            if i % self.partition == 0:
                 try:
-                    self._set_wallets()
+                    self._reuse_wallets() if self.reuse_wallets else self._generate_wallets()
                 except CriticalError:
                     logger.critical(f'Response errors: {self.errors}')
                     break

From cf2618aa39274797b06fb951f49cf8896107d642 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Mon, 13 Jul 2020 14:13:30 -0400
Subject: [PATCH 07/11] New pylint fixes

---
 bin/dev-tools/create_shipments.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 477226b6..482b71be 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -221,7 +221,7 @@ def _retrieve_updated_attributes(self):
                     data={'device_type': "AXLE_GATEWAY"},
                     headers={'Authorization': 'JWT {}'.format(self.shipper['token'])})
             except NonCriticalError:
-                raise CriticalError(f'Error generating device')
+                raise CriticalError('Error generating device')
             updated_attributes['device_id'] = device_response['data']['id']
         self.sequence_number += 1
         return updated_attributes
@@ -309,7 +309,7 @@ def _reuse_storage_credentials(self):
                 f"{self.profiles_url}/api/v1/storage_credentials",
                 headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'}, method='get')
         except NonCriticalError:
-            raise CriticalError(f'Error retrieving storage credentials')
+            raise CriticalError('Error retrieving storage credentials')
 
         logger.debug(f'Storage credentials count returned: {response["meta"]["pagination"]["count"]}')
         if response['meta']['pagination']['count'] == 0:
@@ -328,7 +328,7 @@ def _generate_storage_credentials(self):
                     'title': f'Shipment creator SC: {str(uuid4())}'
                 }, headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
         except NonCriticalError:
-            raise CriticalError(f'Error generating storage credentials.')
+            raise CriticalError('Error generating storage credentials.')
 
         logger.debug(f'Created storage credentials: {response["data"]["id"]}')
         self.attributes['storage_credentials_id'] = response['data']['id']
@@ -398,7 +398,7 @@ def _create_shipment(self):
                 data=attributes,
                 headers={'Authorization': f'JWT {self.get_user_jwt(self.shipper)}'})
         except NonCriticalError:
-            logger.warning(f'Failed to create shipment.')
+            logger.warning('Failed to create shipment.')
             return
 
         logger.debug(f'Created shipment: {response["data"]["id"]}')

From 004936839bbe5db94be2fb411a55c62a061aa408 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Mon, 20 Jul 2020 14:24:28 -0400
Subject: [PATCH 08/11] Move create shipments command to shell and use faker
 for tracking data

---
 bin/create_shipments              |  2 +-
 bin/dev-tools/create_shipments.py | 50 ++++++++++++++++++++++---------
 compose/dev.yml                   |  7 +++++
 pyproject.toml                    |  1 +
 4 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/bin/create_shipments b/bin/create_shipments
index 2dc49050..4c2b11dd 100755
--- a/bin/create_shipments
+++ b/bin/create_shipments
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-bin/dev-tools/create_shipments.py "$@"
+bin/ddo bin/dev-tools/create_shipments.py "$@"
diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 482b71be..898271d6 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -9,6 +9,7 @@
 from uuid import uuid4
 from enum import Enum
 import logging
+from faker import Faker
 
 from copy import deepcopy
 from random import randint
@@ -77,11 +78,11 @@ def __str__(self):
                     action='store_true')
 parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
                     action='store_true')
-parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to http://localhost:9000",
-                    default='http://localhost:9000')
+parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to http://profiles-runserver:8000",
+                    default='http://profiles-runserver:8000')
 parser.add_argument("--transmission_url",
-                    help="Sets the transmission url for the creator. Defaults to http://localhost:8000",
-                    default='http://localhost:8000')
+                    help="Sets the transmission url for the creator. Defaults to http://transmission-runserver:8000",
+                    default='http://transmission-runserver:8000')
 parser.add_argument("--reuse_wallets", help="Reuse wallets instead of creating new ones, defaults to True",
                     action="store_true",
                     default=True)
@@ -112,12 +113,7 @@ class ShipmentCreator:
     }
     tracking_data = {
         "position": {
-            "latitude": -81.048253,
-            "longitude": 34.628643,
-            "altitude": 924,
             "source": "gps",
-            "uncertainty": 95,
-            "speed": 34
         },
         "version": "1.2.4",
         "timestamp": datetime.now(timezone.utc).isoformat(),
@@ -135,15 +131,15 @@ def __init__(self):
 
     # pylint: disable=too-many-arguments, too-many-locals, attribute-defined-outside-init
     def set_shipment_fields(self, carrier='user1@shipchain.io', shipper='user1@shipchain.io', moderator=False,
-                            startnumber=0, total=10, loglevel='warning', device=False, partition=10, attributes=None,
-                            add_tracking=False, add_telemetry=False, profiles_url='http://localhost:9000',
-                            transmission_url='http://localhost:8000', reuse_wallets=False,
+                            startnumber=0, count=10, loglevel='warning', device=False, partition=10, attributes=None,
+                            add_tracking=False, add_telemetry=False, profiles_url='http://profiles-runserver:8000',
+                            transmission_url='http://transmission-runserver:9000', reuse_wallets=False,
                             reuse_storage_credentials=False):
         self.carrier = self.users[carrier]
         self.shipper = self.users[shipper]
         self.moderator = self.users[moderator] if moderator else None
         self.sequence_number = startnumber
-        self.total = total
+        self.total = count
         console.setLevel(LogLevels[loglevel].value)
         logger.addHandler(console)
         logger.setLevel(LogLevels[loglevel].value)
@@ -371,6 +367,7 @@ def _add_telemetry(self, device_id):
             headers={"Content-type": "application/json"}
         )
         if not telemetry_response.ok:
+            self.errors += telemetry_response.json()['errors']
             logger.warning(f'Failed to add telemetry data for device: {device_id}')
         else:
             logger.info(f'Added telemetry data via device: {device_id}')
@@ -378,13 +375,37 @@ def _add_telemetry(self, device_id):
     def _add_tracking(self, device_id):
         logger.info(f'Adding tracking via device: {device_id}')
         self.tracking_data['device_id'] = device_id
-        tracking_data_collection = [{"payload": self.tracking_data} for i in range(10)]
+        faker = Faker()
+        tracking_data_collection = []
+        start_location = faker.local_latlng()
+        end_location = faker.local_latlng()
+        for location in (start_location, end_location):
+            if 'Honolulu' in location[4]:
+                location = faker.local_latlng()
+
+        longitude = float(start_location[0])
+        latitude = float(start_location[1])
+        long_delta = (float(end_location[0]) - longitude) / 10
+        lat_delta = (float(end_location[1]) - latitude) / 10
+        for i in range(10):
+            tracking_data_collection.append({"payload": {
+                'longitude': longitude,
+                'latitude': latitude,
+                'speed': randint(45, 85),
+                'altitude': randint(500, 800),
+                'uncertainty': randint(75, 100),
+                **self.tracking_data
+            }})
+            longitude += long_delta
+            latitude += lat_delta
+
         tracking_response = requests.post(
             f"{self.transmission_url}/api/v1/devices/{device_id}/tracking",
             json=tracking_data_collection,
             headers={"Content-type": "application/json"}
         )
         if not tracking_response.ok:
+            self.errors += tracking_response.json()['errors']
             logger.warning(f'Failed to add tracking data for device: {device_id}')
         else:
             logger.info(f'Added tracking data via device: {device_id}')
@@ -431,6 +452,7 @@ def create_bulk_shipments(self):
 
         logger.info(f'Shipments created: {len(self.shipments)}')
         logger.info(f'Shipment ids: {self.shipments}')
+        logger.info(f'Response errors: {self.errors}')
 
 
 if __name__ == '__main__':
diff --git a/compose/dev.yml b/compose/dev.yml
index 8d35ff3f..f2feb092 100644
--- a/compose/dev.yml
+++ b/compose/dev.yml
@@ -26,6 +26,13 @@ services:
     image: transmission-django-dev
     volumes:
       - ../:/app
+    networks:
+      default:
+        aliases:
+          - transmission-django-shell
+      portal:
+        aliases:
+          - transmission-django-shell
     links:
       - psql
       - redis_db
diff --git a/pyproject.toml b/pyproject.toml
index b54ddc13..2e24a895 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,6 +63,7 @@ uwsgi = "~2.0.17"
 watchdog = "~0.9"
 whitenoise = "~4.1"
 watchtower = "==0.5.5"
+faker = "^4.1.1"
 
 
 [tool.poetry.dev-dependencies]

From 183d3b4d593805eba48c911e42fec6465d7d4ff8 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Mon, 20 Jul 2020 14:31:09 -0400
Subject: [PATCH 09/11] Pylint fixes

---
 bin/dev-tools/create_shipments.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 898271d6..6b93cd01 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -78,7 +78,8 @@ def __str__(self):
                     action='store_true')
 parser.add_argument("--add_telemetry", help="Adds telemetry data to shipments (requires --device or -d)",
                     action='store_true')
-parser.add_argument("--profiles_url", help="Sets the profiles url for the creator. Defaults to http://profiles-runserver:8000",
+parser.add_argument("--profiles_url",
+                    help="Sets the profiles url for the creator. Defaults to http://profiles-runserver:8000",
                     default='http://profiles-runserver:8000')
 parser.add_argument("--transmission_url",
                     help="Sets the transmission url for the creator. Defaults to http://transmission-runserver:8000",

From bb327a343a1ae5748366ebd66a34a2ea0a28a0ed Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Tue, 21 Jul 2020 11:12:51 -0400
Subject: [PATCH 10/11] Change ordering for how token expiration is checked on
 shipment creator

---
 bin/dev-tools/create_shipments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index 6b93cd01..d8425060 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -224,7 +224,7 @@ def _retrieve_updated_attributes(self):
         return updated_attributes
 
     def get_user_jwt(self, user):
-        if user['token_exp'] and user['token_exp'] < datetime.now(timezone.utc):
+        if user['token_exp'] and datetime.now(timezone.utc) < user['token_exp']:
             logger.debug(f'User {user["username"]} has non-expired token')
             return user['token']
 

From 6c3f02d26d8357ae98dec28e009441d0571bdc23 Mon Sep 17 00:00:00 2001
From: James Neyer <jamesneyer@gmail.com>
Date: Fri, 31 Jul 2020 14:55:25 -0400
Subject: [PATCH 11/11] Rebase changes

---
 bin/dev-tools/create_shipments.py | 15 ++++----
 poetry.lock                       | 61 ++++++++++++++++++++++++++-----
 2 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/bin/dev-tools/create_shipments.py b/bin/dev-tools/create_shipments.py
index d8425060..e73eaa15 100755
--- a/bin/dev-tools/create_shipments.py
+++ b/bin/dev-tools/create_shipments.py
@@ -1,19 +1,19 @@
 #! /usr/bin/env python3
 
 import argparse
-import sys
 import json
-from json.decoder import JSONDecodeError
+import logging
+import sys
+from copy import deepcopy
 from datetime import datetime, timezone, timedelta
+from enum import Enum
+from json.decoder import JSONDecodeError
+from random import randint
 from urllib.parse import urlparse
 from uuid import uuid4
-from enum import Enum
-import logging
-from faker import Faker
 
-from copy import deepcopy
-from random import randint
 import requests
+from faker import Faker
 
 # pylint:disable=invalid-name
 logger = logging.getLogger('transmission')
@@ -46,6 +46,7 @@ class LogLevels(Enum):
     info = 'INFO'
     debug = 'DEBUG'
 
+    # pylint: disable=invalid-str-returned
     def __str__(self):
         return self.value
 
diff --git a/poetry.lock b/poetry.lock
index 1fd29b84..f2d12abd 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -16,7 +16,7 @@ description = "Low-level AMQP client for Python (fork of amqplib)."
 name = "amqp"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-version = "2.6.0"
+version = "2.6.1"
 
 [package.dependencies]
 vine = ">=1.1.3,<5.0.0a1"
@@ -863,6 +863,18 @@ optional = false
 python-versions = "*"
 version = "1.0.1"
 
+[[package]]
+category = "main"
+description = "Faker is a Python package that generates fake data for you."
+name = "faker"
+optional = false
+python-versions = ">=3.4"
+version = "4.1.1"
+
+[package.dependencies]
+python-dateutil = ">=2.4"
+text-unidecode = "1.3"
+
 [[package]]
 category = "dev"
 description = "the modular source code checker: pep8 pyflakes and co"
@@ -1074,6 +1086,14 @@ six = ">=1.10.0"
 [package.extras]
 test = ["mock", "nose", "nose-cov", "requests-mock"]
 
+[[package]]
+category = "dev"
+description = "iniconfig: brain-dead simple config-ini parsing"
+name = "iniconfig"
+optional = false
+python-versions = "*"
+version = "1.0.1"
+
 [[package]]
 category = "main"
 description = "Self-contained ISO 3166-1 country definitions."
@@ -1822,24 +1842,25 @@ description = "pytest: simple powerful testing with Python"
 name = "pytest"
 optional = false
 python-versions = ">=3.5"
-version = "5.4.3"
+version = "6.0.1"
 
 [package.dependencies]
 atomicwrites = ">=1.0"
 attrs = ">=17.4.0"
 colorama = "*"
+iniconfig = "*"
 more-itertools = ">=4.0.0"
 packaging = "*"
 pluggy = ">=0.12,<1.0"
-py = ">=1.5.0"
-wcwidth = "*"
+py = ">=1.8.2"
+toml = "*"
 
 [package.dependencies.importlib-metadata]
 python = "<3.8"
 version = ">=0.12"
 
 [package.extras]
-checkqa-mypy = ["mypy (v0.761)"]
+checkqa_mypy = ["mypy (0.780)"]
 testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
 
 [[package]]
@@ -2181,6 +2202,14 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0"
 python = "<3.8"
 version = ">=1.7.0"
 
+[[package]]
+category = "main"
+description = "The most basic Text::Unidecode port"
+name = "text-unidecode"
+optional = false
+python-versions = "*"
+version = "1.3"
+
 [[package]]
 category = "dev"
 description = "module for creating simple ASCII tables"
@@ -2435,7 +2464,7 @@ test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 
 [metadata]
-content-hash = "19562a3c273db2db38c77f8aa6113c2168f779dd2e402aae9d1fb8be09d40fe0"
+content-hash = "ebd7ee19e677945f6ecb3257949142e550d4c952f6a12aa30afc846d1053f84c"
 python-versions = "==3.6.9"
 
 [metadata.files]
@@ -2444,8 +2473,8 @@ aioredis = [
     {file = "aioredis-1.3.1.tar.gz", hash = "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a"},
 ]
 amqp = [
-    {file = "amqp-2.6.0-py2.py3-none-any.whl", hash = "sha256:bb68f8d2bced8f93ccfd07d96c689b716b3227720add971be980accfc2952139"},
-    {file = "amqp-2.6.0.tar.gz", hash = "sha256:24dbaff8ce4f30566bb88976b398e8c4e77637171af3af6f1b9650f48890e60b"},
+    {file = "amqp-2.6.1-py2.py3-none-any.whl", hash = "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"},
+    {file = "amqp-2.6.1.tar.gz", hash = "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21"},
 ]
 argh = [
     {file = "argh-0.26.2-py2.py3-none-any.whl", hash = "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3"},
@@ -2783,6 +2812,10 @@ ecdsa = [
 et-xmlfile = [
     {file = "et_xmlfile-1.0.1.tar.gz", hash = "sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b"},
 ]
+faker = [
+    {file = "Faker-4.1.1-py3-none-any.whl", hash = "sha256:1290f589648bc470b8d98fff1fdff773fe3f46b4ca2cac73ac74668b12cf008e"},
+    {file = "Faker-4.1.1.tar.gz", hash = "sha256:c006b3664c270a2cfd4785c5e41ff263d48101c4e920b5961cf9c237131d8418"},
+]
 flake8 = [
     {file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"},
     {file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"},
@@ -2913,6 +2946,10 @@ influxdb = [
     {file = "influxdb-5.0.0-py2.py3-none-any.whl", hash = "sha256:43b2fde195ee2302cfa87c8a0e1d29429f175ed584516d02e04f9b2c3c2ac2ad"},
     {file = "influxdb-5.0.0.tar.gz", hash = "sha256:6adba2ddfd5781a06b5204339e679d66645bf6cc2b7f493eb9d7c8986d714e80"},
 ]
+iniconfig = [
+    {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"},
+    {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"},
+]
 iso3166 = [
     {file = "iso3166-1.0.1-py2.py3-none-any.whl", hash = "sha256:b07208703bd881a4f974e39fa013c4498dddd64913ada15f24be75d02ae68a44"},
     {file = "iso3166-1.0.1.tar.gz", hash = "sha256:b1e58dbcf50fbb2c9c418ec7a6057f0cdb30b8f822ac852f72e71ba769dae8c5"},
@@ -3276,8 +3313,8 @@ pyrsistent = [
     {file = "pyrsistent-0.16.0.tar.gz", hash = "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3"},
 ]
 pytest = [
-    {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
-    {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
+    {file = "pytest-6.0.1-py3-none-any.whl", hash = "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad"},
+    {file = "pytest-6.0.1.tar.gz", hash = "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4"},
 ]
 pytest-asyncio = [
     {file = "pytest-asyncio-0.14.0.tar.gz", hash = "sha256:9882c0c6b24429449f5f969a5158b528f39bde47dc32e85b9f0403965017e700"},
@@ -3403,6 +3440,10 @@ stevedore = [
     {file = "stevedore-3.2.0-py3-none-any.whl", hash = "sha256:c8f4f0ebbc394e52ddf49de8bcc3cf8ad2b4425ebac494106bbc5e3661ac7633"},
     {file = "stevedore-3.2.0.tar.gz", hash = "sha256:38791aa5bed922b0a844513c5f9ed37774b68edc609e5ab8ab8d8fe0ce4315e5"},
 ]
+text-unidecode = [
+    {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"},
+    {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"},
+]
 texttable = [
     {file = "texttable-1.6.2-py2.py3-none-any.whl", hash = "sha256:7dc282a5b22564fe0fdc1c771382d5dd9a54742047c61558e071c8cd595add86"},
     {file = "texttable-1.6.2.tar.gz", hash = "sha256:eff3703781fbc7750125f50e10f001195174f13825a92a45e9403037d539b4f4"},