MPT-6862 support url for mongodb

arcee/arcee_receiver/server.py

@@ -1,7 +1,8 @@
+import asyncio
 import time
 from collections import OrderedDict, defaultdict
 from datetime import datetime, timezone, timedelta
-import asyncio
+from urllib.parse import urlparse
 
 from etcd import Lock as EtcdLock, Client as EtcdClient
 from typing import Tuple
@@ -60,7 +61,7 @@ async def add_spent_time(request, response):
                                           spend_time))
 
 
-def get_arcee_db_params() -> Tuple[str, str, str, str, str]:
+def get_arcee_db_params() -> Tuple[str, str]:
     arcee_db_params = config_client.arcee_params()
     return asyncio.run(arcee_db_params)
 
@@ -69,9 +70,12 @@ async def get_cluster_secret() -> str:
     return await config_client.cluster_secret()
 
 
-name, password, host, port, db_name = get_arcee_db_params()
-uri = "mongodb://{u}:{p}@{host}:{port}/admin".format(
-    u=name, p=password, host=host, port=port)
+conn_url, db_name = get_arcee_db_params()
+components = urlparse(conn_url)
+user_data, host_data = components.netloc.split("@", 1)
+name, password = user_data.split(":", 1)
+host, port = host_data.split(":", 1)
+uri = f"{components.scheme}://{components.netloc}/admin?{components.query}"
 client = motor.motor_asyncio.AsyncIOMotorClient(uri)
 # https://stackoverflow.com/a/69065287
 client.get_io_loop = asyncio.get_running_loop
@@ -2699,11 +2703,11 @@ async def delete_artifact(request, id_: str):
     # trick to lock migrations
     with EtcdLock(
             EtcdClient(host=etcd_host, port=etcd_port), 'arcee_migrations'):
+
         config_params = {
             'mongo_username': name,
             'mongo_password': password,
-            'mongo_url': "mongodb://{host}:{port}/admin".format(
-                host=host, port=port),
+            'mongo_url': uri,
             'mongo_database': db_name,
             'mongo_migrations_path': os.path.join(
                 os.path.dirname(os.path.abspath(__file__)), 'migrations')

arcee/arcee_receiver/tests/base.py

@@ -14,7 +14,7 @@
 class AConfigClMock(AConfigCl):
 
     async def arcee_params(self):
-        return 'name', 'password', '127.0.0.1', 80, 'arcee'
+        return 'mongodb://name:password@127.0.0.1:80', 'arcee'
 
     async def cluster_secret(self):
         return SECRET

bi_exporter/bumblebi/exporter/exporter.py

@@ -45,8 +45,7 @@ def rest_cl(self) -> RestClient:
     @cached_property
     def mongo_cl(self) -> MongoClient:
         mongo_params = self._config_cl.mongo_params()
-        mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-        return MongoClient(mongo_conn_string)
+        return MongoClient(mongo_params[0])
 
     @cached_property
     def clickhouse_cl(self) -> ClickHouseClient:

bulldozer/bulldozer_api/server.py

@@ -1,6 +1,7 @@
 import datetime
 import asyncio
 from typing import Tuple
+from urllib.parse import urlparse
 from enum import Enum
 import os
 import uuid
@@ -56,7 +57,7 @@ class RunsetState(int, Enum):
 MAX_RUNNER_NUM = 15
 
 
-def get_db_params() -> Tuple[str, str, str, str, str]:
+def get_db_params() -> Tuple[str, str]:
     db_params = config_client.bulldozer_params()
     return asyncio.run(db_params)
 
@@ -65,8 +66,12 @@ async def get_cluster_secret() -> str:
     return await config_client.cluster_secret()
 
 
-name, password, host, port, db_name = get_db_params()
-uri = f"mongodb://{name}:{password}@{host}:{port}/admin"
+conn_url, db_name = get_db_params()
+components = urlparse(conn_url)
+user_data, host_data = components.netloc.split("@", 1)
+name, password = user_data.split(":", 1)
+host, port = host_data.split(":", 1)
+uri = f"{components.scheme}://{components.netloc}/admin?{components.query}"
 client = motor.motor_asyncio.AsyncIOMotorClient(uri)
 db = client[db_name]
 
@@ -930,8 +935,7 @@ async def update_runner(request, id_: str):
         config_params = {
             'mongo_username': name,
             'mongo_password': password,
-            'mongo_url': "mongodb://{host}:{port}/admin".format(
-                host=host, port=port),
+            'mongo_url': uri,
             'mongo_database': db_name
         }
         manager = MigrationManager(config=Configuration(config=config_params))

bumiworker/bumiworker/modules/base.py

@@ -70,8 +70,7 @@ def rest_client(self):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     def get_options(self):

bumiworker/bumiworker/tasks.py

@@ -80,8 +80,7 @@ def s3_client(self):
     def mongo_cl(self):
         if not self._mongo_cl:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     def _execute(self):

diworker/diworker/main.py

@@ -70,8 +70,7 @@ def rest_cl(self):
     def mongo_cl(self):
         if self._mongo_cl is None:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     @property

diworker/diworker/migrator.py

@@ -32,8 +32,7 @@ def __init__(self, config_cl, database_name, migrations_path):
         self.migrations_path = migrations_path
 
         mongo_params = self.config_cl.mongo_params()
-        mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-        self._mongo_cl = MongoClient(mongo_conn_string)
+        self._mongo_cl = MongoClient(mongo_params[0])
 
     @property
     def _mongo_migrations(self):

diworker/diworker/regenerator.py

@@ -26,8 +26,7 @@ def rest_cl(self):
     def mongo_cl(self):
         if self._mongo_cl is None:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     @property

docker_images/booking_observer/worker.py

@@ -51,8 +51,7 @@ def rest_cl(self):
     def mongo_cl(self):
         if not self._mongo_cl:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     def get_consumers(self, consumer, channel):

docker_images/cleanmongodb/clean-mongo-db.py

@@ -71,8 +71,7 @@ def config_client(self):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self.config_client.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     def get_settings(self):

docker_images/configurator/configurator.py

@@ -35,10 +35,14 @@ def __init__(self, config_path='config.yml', host='etcd', port=2379):
             host=config['restdb']['host'],
             port=config['restdb']['port'])
         )
-        self.mongo_client = MongoClient("mongodb://%s:%s@%s:%s" % (
-            config['mongo']['user'], config['mongo']['pass'],
-            config['mongo']['host'], config['mongo']['port']
-        ))
+        if "url" in config["mongo"]:
+            mongo_url = config["mongo"]["url"]
+        else:
+            mongo_url = "mongodb://%s:%s@%s:%s" % (
+                config['mongo']['user'], config['mongo']['pass'],
+                config['mongo']['host'], config['mongo']['port']
+            )
+        self.mongo_client = MongoClient(mongo_url)
 
         rabbit_config = config['rabbit']
         credentials = pika.PlainCredentials(rabbit_config['user'],

docker_images/live_demo_generator/scheduler.py

@@ -36,8 +36,7 @@ def publish_tasks(config_client, count):
 
 def main(config_client):
     mongo_params = config_client.mongo_params()
-    mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-    mongo_cl = MongoClient(mongo_conn_string)
+    mongo_cl = MongoClient(mongo_params[0])
     live_demos_collection = mongo_cl.restapi.live_demos
     dt = utcnow() - timedelta(days=DEMO_LIFETIME_DAYS)
     count = DEMO_COUNT - live_demos_collection.count_documents({

docker_images/live_demo_generator/worker.py

@@ -46,8 +46,7 @@ def rest_cl(self):
     def mongo_cl(self):
         if not self._mongo_cl:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     def get_consumers(self, Consumer, channel):

docker_images/power_schedule/worker.py

@@ -64,8 +64,7 @@ def rest_cl(self):
     @cached_property
     def mongo_cl(self):
         mongo_params = self.config_cl.mongo_params()
-        mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-        return MongoClient(mongo_conn_string)
+        return MongoClient(mongo_params[0])
 
     def get_consumers(self, consumer, channel):
         return [consumer(queues=[TASK_QUEUE], accept=['json'],

docker_images/users_dataset_generator/users_dataset_generator.py

@@ -42,10 +42,8 @@ def _get_session_to_my_db(config_cl):
 
 
 def _get_mongo_client(config_cl):
-    user, password, host, port, db_name = config_cl.mongo_params()
-    mongo_conn_string = "mongodb://%s:%s@%s:%s" % (
-        user, password, host, port)
-    return MongoClient(mongo_conn_string)
+    mongo_params = config_cl.mongo_params()
+    return MongoClient(mongo_params[0])
 
 
 def _get_clickhouse_client(config_cl):

docker_images/webhook_executor/worker.py

@@ -82,8 +82,7 @@ def rest_cl(self):
     def mongo_cl(self):
         if not self._mongo_cl:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_cl = MongoClient(mongo_conn_string)
+            self._mongo_cl = MongoClient(mongo_params[0])
         return self._mongo_cl
 
     def get_consumers(self, consumer, channel):

gemini/gemini_worker/main.py

@@ -60,8 +60,7 @@ def clickhouse_client(self) -> ClickHouseClient:
     def mongo_client(self) -> MongoClient:
         if self._mongo_client is None:
             mongo_params = self.config_client.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

insider/insider_api/controllers/base.py

@@ -65,8 +65,7 @@ def __init__(self, config=None):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self._config.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

insider/insider_api/controllers/flavor_price.py

@@ -29,8 +29,7 @@ def __init__(self, config_cl):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self._config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

insider/insider_api/controllers/relevant_flavor.py

@@ -30,8 +30,7 @@ def __init__(self, config_cl):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self._config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

insider/insider_worker/main.py

@@ -33,8 +33,7 @@ def __init__(self, connection, config_cl):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

insider/insider_worker/migrator.py

@@ -32,8 +32,7 @@ def __init__(self, config_cl, database_name, migrations_path):
         self.migrations_path = migrations_path
 
         mongo_params = self.config_cl.mongo_params()
-        mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-        self._mongo_cl = MongoClient(mongo_conn_string)
+        self._mongo_cl = MongoClient(mongo_params[0])
 
     @property
     def _mongo_migrations(self):

metroculus/metroculus_worker/processor.py

@@ -115,8 +115,7 @@ def clickhouse_client(self):
     def mongo_client(self):
         if not self._mongo_client:
             mongo_params = self.config_cl.mongo_params()
-            mongo_conn_string = "mongodb://%s:%s@%s:%s" % mongo_params[:-1]
-            self._mongo_client = MongoClient(mongo_conn_string)
+            self._mongo_client = MongoClient(mongo_params[0])
         return self._mongo_client
 
     @property

optscale-deploy/optscale/templates/tpl/_config.tpl

@@ -156,10 +156,11 @@ etcd:
     db: jira-bus
     port: {{ .Values.mariadb.service.externalPort }}
   mongo:
-    host: {{ .Values.mongo.service.host }}
-    port: {{ .Values.mongo.service.externalPort }}
-    user: {{ .Values.mongo.credentials.username }}
-    pass: {{ .Values.mongo.credentials.password }}
+    {{ if .Values.mongo.url }}
+      url: {{ .Values.mongo.url }}
+    {{ else }}
+      url: mongodb://{{ .Values.mongo.credentials.username }}:{{ .Values.mongo.credentials.password }}@{{ .Values.mongo.service.host }}:{{ .Values.mongo.service.externalPort }}
+    {{ end }}
     database: keeper
   influxdb:
     host: {{ .Values.influxdb.service.name }}

optscale_client/aconfig_cl/aconfig_cl.py

@@ -28,18 +28,28 @@ async def arcee_params(self):
         Returns
         :return:
         """
-        branch = await self.read_branch("/mongo")
-        return (branch['user'], branch['pass'], branch['host'],
-                branch['port'], 'arcee')
+        params = self.read_branch('/mongo')
+        if "url" in params:
+            return (params["url"], 'arcee')
+        url = "mongodb://%s:%s@%s:%s"  % (
+            params['user'], params['pass'],
+            params['host'], params['port']
+        )
+        return (url, 'arcee')
 
     async def bulldozer_params(self):
         """
         Returns
         :return:
         """
-        branch = await self.read_branch("/mongo")
-        return (branch['user'], branch['pass'], branch['host'],
-                branch['port'], 'bulldozer')
+        params = self.read_branch('/mongo')
+        if "url" in params:
+            return (params["url"], 'bulldozer')
+        url = "mongodb://%s:%s@%s:%s"  % (
+            params['user'], params['pass'],
+            params['host'], params['port']
+        )
+        return (url, 'bulldozer')
 
     async def cluster_secret(self):
         return await self.get_value("/secret/cluster")

optscale_client/config_client/client.py

@@ -343,13 +343,14 @@ def increase_and_return(self, key, max_value=50000):
             raise ConcurrencyException(key)
 
     def mongo_params(self):
-        """
-        Get tuple with access args for mongo db (report service)
-        :return: ('user', 'pass', 'host', 'port', 'database')
-        """
-        params = self.read_branch('/mongo')
-        return (params['user'], params['pass'], params['host'],
-                params['port'], params['database'])
+         params = self.read_branch('/mongo')
+         if "url" in params:
+             return (params["url"], params["database"])
+         url = "mongodb://%s:%s@%s:%s"  % (
+            params['user'], params['pass'],
+            params['host'], params['port']
+         )
+         return (url, params["database"])
 
     def rabbit_params(self):
         """