From 5b6b26fa2ac6275e2b159dc9a6b334c4124874b2 Mon Sep 17 00:00:00 2001
From: Pietro Pasotti <starfire.daemon@gmail.com>
Date: Fri, 5 May 2023 10:25:02 +0200
Subject: [PATCH 1/5] base mongo wrapper working

---
 jhack/helpers.py                              |  8 ++
 jhack/mongo/eson.py                           | 26 ++++++
 .../get_credentials_from_k8s_controller.sh    |  8 ++
 ...get_credentials_from_machine_controller.sh | 19 +++++
 jhack/mongo/mongo.py                          | 84 +++++++++++++++++++
 jhack/mongo/query_k8s_controller.sh           |  8 ++
 jhack/mongo/query_machine_controller.sh       |  8 ++
 .../tests/mongo/test_k8s_connector_manual.py  |  8 ++
 .../mongo/test_machine_connector_manual.py    |  8 ++
 9 files changed, 177 insertions(+)
 create mode 100644 jhack/mongo/eson.py
 create mode 100644 jhack/mongo/get_credentials_from_k8s_controller.sh
 create mode 100644 jhack/mongo/get_credentials_from_machine_controller.sh
 create mode 100644 jhack/mongo/mongo.py
 create mode 100644 jhack/mongo/query_k8s_controller.sh
 create mode 100644 jhack/mongo/query_machine_controller.sh
 create mode 100644 jhack/tests/mongo/test_k8s_connector_manual.py
 create mode 100644 jhack/tests/mongo/test_machine_connector_manual.py

diff --git a/jhack/helpers.py b/jhack/helpers.py
index dc861cd..8431ae4 100644
--- a/jhack/helpers.py
+++ b/jhack/helpers.py
@@ -39,6 +39,14 @@ def check_command_available(cmd: str):
     return proc.returncode == 0
 
 
+def get_current_controller() -> str:
+    cmd = f'juju whoami --format=json'
+    proc = JPopen(cmd.split())
+    raw = proc.stdout.read().decode("utf-8")
+    whoami_info = jsn.loads(raw)
+    return whoami_info['controller']
+
+
 def get_substrate(model: str = None) -> Literal["k8s", "machine"]:
     """Attempts to guess whether we're talking k8s or machine."""
     cmd = f'juju show-model{f" {model}" if model else ""} --format=json'
diff --git a/jhack/mongo/eson.py b/jhack/mongo/eson.py
new file mode 100644
index 0000000..9051d5d
--- /dev/null
+++ b/jhack/mongo/eson.py
@@ -0,0 +1,26 @@
+def emit_thing(v):
+    if v.__class__.__name__ == 'dict':
+        return emit_dict(v)
+    elif v.__class__.__name__ == 'list':
+        return emit_list(v)
+    elif v.__class__.__name__ in {'Int64', 'int', 'long', 'float', 'decimal', 'Decimal128', 'Decimal'}:
+        return str(v)
+    elif v.__class__.__name__ == 'datetime':
+        return v
+    else:
+        return str(v)
+
+
+def emit_list(ll: list) -> list:
+    return list(map(emit_thing, ll))
+
+
+def emit_dict(dd: dict) -> dict:
+    out = {}
+    for k, v in dd.items():
+        out[k] = emit_thing(v)
+    return out
+
+
+def parse_eson(doc: str) -> dict:
+    return emit_dict()
diff --git a/jhack/mongo/get_credentials_from_k8s_controller.sh b/jhack/mongo/get_credentials_from_k8s_controller.sh
new file mode 100644
index 0000000..4b052a3
--- /dev/null
+++ b/jhack/mongo/get_credentials_from_k8s_controller.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+kubectl_bin=microk8s.kubectl
+k8s_ns=`juju whoami | grep Controller | awk '{print "controller-"$2}'`
+k8s_controller_pod=`${kubectl_bin} -n ${k8s_ns} get pods | grep -E "^controller-([0-9]+)" | awk '{print $1}'`
+mongo_user=`${kubectl_bin} exec -n ${k8s_ns} ${k8s_controller_pod} -c api-server -it -- bash -c "grep tag /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'"`
+mongo_pass=`${kubectl_bin} exec -n ${k8s_ns} ${k8s_controller_pod} -c api-server -it -- bash -c "grep statepassword /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'"`
+
+echo "$k8s_ns" "$k8s_controller_pod" --password "$mongo_pass" --username "$mongo_user"
diff --git a/jhack/mongo/get_credentials_from_machine_controller.sh b/jhack/mongo/get_credentials_from_machine_controller.sh
new file mode 100644
index 0000000..a417c07
--- /dev/null
+++ b/jhack/mongo/get_credentials_from_machine_controller.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+machine=${1:-0}
+model=${2:-controller}
+
+read -d '' -r cmds <<'EOF'
+conf=/var/lib/juju/agents/machine-*/agent.conf
+user=`sudo grep tag $conf | cut -d' ' -f2`
+password=`sudo grep statepassword $conf | cut -d' ' -f2`
+if [ -f /snap/bin/juju-db.mongo ]; then
+  client=/snap/bin/juju-db.mongo
+elif [ -f /usr/lib/juju/mongo*/bin/mongo ]; then
+  client=/usr/lib/juju/mongo*/bin/mongo
+else
+  client=/usr/bin/mongo
+fi
+echo "$client" "$user" "$password"
+EOF
+juju ssh -m "${model}" "${machine}" "${cmds}"
diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py
new file mode 100644
index 0000000..7d02d01
--- /dev/null
+++ b/jhack/mongo/mongo.py
@@ -0,0 +1,84 @@
+#!/bin/bash
+import json
+import re
+import shlex
+from pathlib import Path
+from subprocess import Popen, PIPE
+from typing import Literal, Tuple
+
+from jhack.helpers import get_current_model, get_substrate, JPopen
+
+
+def escape_double_quotes(query):
+    return query.replace('"', r'\"')
+
+
+class ConnectorBase:
+    args_getter_script: Path
+    query_script: Path
+
+    def __init__(self):
+        self.args = self.get_args()
+
+    def _get_output(self, cmd: str) -> str:
+        return JPopen(shlex.split(cmd)).stdout.read().decode('utf-8')
+
+    def get_args(self) -> Tuple[str, ...]:
+        out = Popen(
+            shlex.split(
+                f"bash {self.args_getter_script.absolute()}",
+            ), stdout=PIPE
+        ).stdout.read().decode('utf-8')
+        return tuple(f"'{x}'" for x in out.split())  # noqa
+
+    def query(self, query: str):
+        query = escape_double_quotes(query)
+        command = ["bash", str(self.query_script.absolute()), *self.args, f""" "'{query}'" """]
+        proc = Popen(command, stdout=PIPE, stderr=PIPE)
+        out = proc.stdout.read().decode('utf-8')
+        txt = out.split('\n')
+        if len(txt) != 3:
+            raise RuntimeError(f'unexpected result from command {command}; {proc.stderr.read()}')
+        result = txt[1]
+        return parse_eson(result)
+
+
+class K8sConnector(ConnectorBase):
+    """Mongo database connector for kubernetes controllers."""
+    args_getter_script = Path(__file__).parent / 'get_credentials_from_k8s_controller.sh'
+    query_script = Path(__file__).parent / 'query_k8s_controller.sh'
+
+    def get_args(self):
+        return ("microk8s.kubectl", ) + super().get_args()
+
+
+
+class MachineConnector(ConnectorBase):
+    """Mongo database connector for kubernetes controllers."""
+    args_getter_script = Path(__file__).parent / 'get_credentials_from_machine_controller.sh'
+    query_script = Path(__file__).parent / 'query_machine_controller.sh'
+
+    def get_args(self):
+        return super().get_args() + ("controller", "0")
+
+
+class Mongo:
+    def __init__(self,
+                 entity_id: int = 0,
+                 substrate: Literal['k8s', 'machine'] = None,
+                 model: str = None):
+        self.substrate = substrate or get_substrate()
+        self.entity_id = entity_id
+        self.model = model or get_current_model()
+
+        if substrate == 'k8s':
+            self.connector = K8sConnector()
+
+        elif substrate == 'machine':
+            self.connector = MachineConnector()
+
+        else:
+            raise TypeError(substrate)
+
+    def query(self, q: str):
+        return self.connector.query(q)
diff --git a/jhack/mongo/query_k8s_controller.sh b/jhack/mongo/query_k8s_controller.sh
new file mode 100644
index 0000000..0d970d0
--- /dev/null
+++ b/jhack/mongo/query_k8s_controller.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+kctl=${1}
+user=${2}
+password=${3}
+k8s_ns=${4}
+k8s_controller_pod=${5}
+query=${6}
+${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'"
diff --git a/jhack/mongo/query_machine_controller.sh b/jhack/mongo/query_machine_controller.sh
new file mode 100644
index 0000000..15f7684
--- /dev/null
+++ b/jhack/mongo/query_machine_controller.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+client=${1}
+user=${2}
+password=${3}
+controller=${4}
+machine=${5}
+query=${6}
+juju ssh -m "$controller" "$machine" -- "$client" '127.0.0.1:37017/juju' --authenticationDatabase admin --tls --tlsAllowInvalidCertificates --quiet --username "$user" --password "$password" --eval "$query"
diff --git a/jhack/tests/mongo/test_k8s_connector_manual.py b/jhack/tests/mongo/test_k8s_connector_manual.py
new file mode 100644
index 0000000..e8db225
--- /dev/null
+++ b/jhack/tests/mongo/test_k8s_connector_manual.py
@@ -0,0 +1,8 @@
+from jhack.mongo.mongo import K8sConnector
+
+
+def test_k8s_connector():
+    connector = K8sConnector()
+    query = 'db.relations.find({"key": "loki:logging"}).pretty()'
+    val = connector.query(query)
+    assert val
diff --git a/jhack/tests/mongo/test_machine_connector_manual.py b/jhack/tests/mongo/test_machine_connector_manual.py
new file mode 100644
index 0000000..109b784
--- /dev/null
+++ b/jhack/tests/mongo/test_machine_connector_manual.py
@@ -0,0 +1,8 @@
+from jhack.mongo.mongo import MachineConnector
+
+
+def test_k8s_connector():
+    connector = MachineConnector()
+    query = 'db.relations.find({"key": "kafka:cluster"})'
+    val = connector.query(query)
+    assert val

From fb97e48f9cb8bb69ca594f4ec2a99462bda9b2eb Mon Sep 17 00:00:00 2001
From: Pietro Pasotti <starfire.daemon@gmail.com>
Date: Fri, 5 May 2023 13:31:37 +0200
Subject: [PATCH 2/5] mongo connection

---
 jhack/conf/conf.py                            |   6 +-
 jhack/helpers.py                              |   4 +-
 jhack/mongo/eson.py                           |  38 +++++-
 .../get_credentials_from_k8s_controller.sh    |  12 +-
 ...get_credentials_from_machine_controller.sh |   4 +-
 jhack/mongo/mongo.py                          | 113 +++++++++++++-----
 jhack/mongo/query_k8s_controller.sh           |   6 +-
 jhack/mongo/query_machine_controller.sh       |   5 +-
 jhack/tests/config/test_config.py             |   1 -
 .../tests/mongo/test_k8s_connector_manual.py  |  11 +-
 .../mongo/test_machine_connector_manual.py    |  11 +-
 11 files changed, 154 insertions(+), 57 deletions(-)

diff --git a/jhack/conf/conf.py b/jhack/conf/conf.py
index acb686a..aff0b30 100644
--- a/jhack/conf/conf.py
+++ b/jhack/conf/conf.py
@@ -58,10 +58,12 @@ def get(self, *path: str) -> bool:  # todo: add more toml types?
                 data = data[item]
             except KeyError:
                 if self._path is self._DEFAULTS:
-                    logger.error(f'{item} not found in default config; invalid path')
+                    logger.error(f"{item} not found in default config; invalid path")
                     raise
 
-                logger.info(f'{item} not found in user-config {self._path}; defaulting...')
+                logger.info(
+                    f"{item} not found in user-config {self._path}; defaulting..."
+                )
                 return self.get_default(*path)
         return data
 
diff --git a/jhack/helpers.py b/jhack/helpers.py
index 8431ae4..b2bc918 100644
--- a/jhack/helpers.py
+++ b/jhack/helpers.py
@@ -40,11 +40,11 @@ def check_command_available(cmd: str):
 
 
 def get_current_controller() -> str:
-    cmd = f'juju whoami --format=json'
+    cmd = f"juju whoami --format=json"
     proc = JPopen(cmd.split())
     raw = proc.stdout.read().decode("utf-8")
     whoami_info = jsn.loads(raw)
-    return whoami_info['controller']
+    return whoami_info["controller"]
 
 
 def get_substrate(model: str = None) -> Literal["k8s", "machine"]:
diff --git a/jhack/mongo/eson.py b/jhack/mongo/eson.py
index 9051d5d..b4ad0ca 100644
--- a/jhack/mongo/eson.py
+++ b/jhack/mongo/eson.py
@@ -1,11 +1,19 @@
 def emit_thing(v):
-    if v.__class__.__name__ == 'dict':
+    if v.__class__.__name__ == "dict":
         return emit_dict(v)
-    elif v.__class__.__name__ == 'list':
+    elif v.__class__.__name__ == "list":
         return emit_list(v)
-    elif v.__class__.__name__ in {'Int64', 'int', 'long', 'float', 'decimal', 'Decimal128', 'Decimal'}:
+    elif v.__class__.__name__ in {
+        "Int64",
+        "int",
+        "long",
+        "float",
+        "decimal",
+        "Decimal128",
+        "Decimal",
+    }:
         return str(v)
-    elif v.__class__.__name__ == 'datetime':
+    elif v.__class__.__name__ == "datetime":
         return v
     else:
         return str(v)
@@ -23,4 +31,24 @@ def emit_dict(dd: dict) -> dict:
 
 
 def parse_eson(doc: str) -> dict:
-    return emit_dict()
+    return emit_dict(doc)
+
+
+import json
+import re
+
+from bson import json_util
+
+
+def read_mongoextjson_file(filename):
+    with open(filename, "r") as f:
+        bsondata = f.read()
+        # Convert Mongo object(s) to regular strict JSON
+        jsondata = re.sub(
+            r"ObjectId\s*\(\s*\"(\S+)\"\s*\)", r'{"$oid": "\1"}', bsondata
+        )
+        # Description of Mongo ObjectId:
+        # https://docs.mongodb.com/manual/reference/mongodb-extended-json/#mongodb-bsontype-ObjectId
+        # now we can parse this as JSON, and use MongoDB's object_hook
+        data = json.loads(jsondata, object_hook=json_util.object_hook)
+        return data
diff --git a/jhack/mongo/get_credentials_from_k8s_controller.sh b/jhack/mongo/get_credentials_from_k8s_controller.sh
index 4b052a3..641addf 100644
--- a/jhack/mongo/get_credentials_from_k8s_controller.sh
+++ b/jhack/mongo/get_credentials_from_k8s_controller.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
-kubectl_bin=microk8s.kubectl
-k8s_ns=`juju whoami | grep Controller | awk '{print "controller-"$2}'`
-k8s_controller_pod=`${kubectl_bin} -n ${k8s_ns} get pods | grep -E "^controller-([0-9]+)" | awk '{print $1}'`
-mongo_user=`${kubectl_bin} exec -n ${k8s_ns} ${k8s_controller_pod} -c api-server -it -- bash -c "grep tag /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'"`
-mongo_pass=`${kubectl_bin} exec -n ${k8s_ns} ${k8s_controller_pod} -c api-server -it -- bash -c "grep statepassword /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'"`
+kubectl_bin=/snap/bin/microk8s.kubectl
+k8s_ns=$(juju whoami | grep Controller | awk '{print "controller-"$2}')
+k8s_controller_pod=$(${kubectl_bin} -n "${k8s_ns}" get pods | grep -E "^controller-([0-9]+)" | awk '{print $1}')
+mongo_user=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep tag /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'")
+mongo_pass=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep statepassword /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'")
 
-echo "$k8s_ns" "$k8s_controller_pod" --password "$mongo_pass" --username "$mongo_user"
+echo "$kubectl_bin" "$mongo_user" "$mongo_pass" "$k8s_ns" "$k8s_controller_pod"
diff --git a/jhack/mongo/get_credentials_from_machine_controller.sh b/jhack/mongo/get_credentials_from_machine_controller.sh
index a417c07..229d768 100644
--- a/jhack/mongo/get_credentials_from_machine_controller.sh
+++ b/jhack/mongo/get_credentials_from_machine_controller.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-machine=${1:-0}
-model=${2:-controller}
+machine=${1}
+model=${2}
 
 read -d '' -r cmds <<'EOF'
 conf=/var/lib/juju/agents/machine-*/agent.conf
diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py
index 7d02d01..c406974 100644
--- a/jhack/mongo/mongo.py
+++ b/jhack/mongo/mongo.py
@@ -1,80 +1,129 @@
 #!/bin/bash
 import json
+import os
 import re
 import shlex
 from pathlib import Path
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
 from typing import Literal, Tuple
 
-from jhack.helpers import get_current_model, get_substrate, JPopen
+from jhack.helpers import JPopen, get_current_model, get_substrate
 
 
 def escape_double_quotes(query):
-    return query.replace('"', r'\"')
+    return query.replace('"', r"\"")
+
+
+def numberlong(s: str):
+    return re.sub(r"NumberLong\((\d+)\)", r"\1", s)
+
+
+FILTERS = [numberlong]
+
+
+def to_json(query_result: str):
+    jsn_str = query_result
+    for f in FILTERS:
+        jsn_str = f(jsn_str)
+    return json.loads(jsn_str)
 
 
 class ConnectorBase:
     args_getter_script: Path
     query_script: Path
 
-    def __init__(self):
+    def __init__(self, controller: str = None, unit_id: int = 0):
+        self.controller = controller
+        self.model = f"{controller}:controller" if controller else "controller"
+        self.unit_id = unit_id
         self.args = self.get_args()
 
+    def _escape_query(self, query: str) -> str:
+        return rf'"{escape_double_quotes(query)}"'
+
     def _get_output(self, cmd: str) -> str:
-        return JPopen(shlex.split(cmd)).stdout.read().decode('utf-8')
+        return JPopen(shlex.split(cmd)).stdout.read().decode("utf-8")
 
     def get_args(self) -> Tuple[str, ...]:
-        out = Popen(
-            shlex.split(
-                f"bash {self.args_getter_script.absolute()}",
-            ), stdout=PIPE
-        ).stdout.read().decode('utf-8')
-        return tuple(f"'{x}'" for x in out.split())  # noqa
+        out = (
+            Popen(
+                shlex.split(
+                    f"bash {self.args_getter_script.absolute()} {self.unit_id} {self.model}",
+                ),
+                stdout=PIPE,
+            )
+            .stdout.read()
+            .decode("utf-8")
+        )
+        return tuple(out.split())  # noqa
+        # return tuple(f"'{x}'" for x in out.split())  # noqa
 
     def query(self, query: str):
-        query = escape_double_quotes(query)
-        command = ["bash", str(self.query_script.absolute()), *self.args, f""" "'{query}'" """]
+        if "pretty()" in query:
+            # we need one result per line to be able to deserialize.
+            raise ValueError("invalid query: unsupported pretty() statement")
+        raw = self._escape_query(query)
+        return self._run_query(raw)
+
+    def _run_query(self, query: str):
+        command = ["bash", str(self.query_script.absolute()), *self.args, query]
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
-        out = proc.stdout.read().decode('utf-8')
-        txt = out.split('\n')
-        if len(txt) != 3:
-            raise RuntimeError(f'unexpected result from command {command}; {proc.stderr.read()}')
+        out = proc.stdout.read().decode("utf-8")
+        if not out:
+            err = proc.stderr.read().decode("utf-8")
+            print(err)
+            raise RuntimeError(f"unexpected result from command {command}; {err!r}")
+
+        txt = out.split("\n")
+        # todo: some queries return a list of documents.
         result = txt[1]
-        return parse_eson(result)
+
+        try:
+            return to_json(result)
+        except Exception as e:
+            err = proc.stderr.read().decode("utf-8")
+            print(err)
+            raise RuntimeError(
+                f"failed deserializing query result {result} with {type(e)} {err}"
+            ) from e
 
 
 class K8sConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
-    args_getter_script = Path(__file__).parent / 'get_credentials_from_k8s_controller.sh'
-    query_script = Path(__file__).parent / 'query_k8s_controller.sh'
-
-    def get_args(self):
-        return ("microk8s.kubectl", ) + super().get_args()
 
+    args_getter_script = (
+        Path(__file__).parent / "get_credentials_from_k8s_controller.sh"
+    )
+    query_script = Path(__file__).parent / "query_k8s_controller.sh"
 
 
 class MachineConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
-    args_getter_script = Path(__file__).parent / 'get_credentials_from_machine_controller.sh'
-    query_script = Path(__file__).parent / 'query_machine_controller.sh'
+
+    args_getter_script = (
+        Path(__file__).parent / "get_credentials_from_machine_controller.sh"
+    )
+    query_script = Path(__file__).parent / "query_machine_controller.sh"
 
     def get_args(self):
-        return super().get_args() + ("controller", "0")
+        return super().get_args() + (self.model, str(self.unit_id))
 
 
 class Mongo:
-    def __init__(self,
-                 entity_id: int = 0,
-                 substrate: Literal['k8s', 'machine'] = None,
-                 model: str = None):
+    def __init__(
+        self,
+        entity_id: int = 0,
+        substrate: Literal["k8s", "machine"] = None,
+        model: str = None,
+    ):
         self.substrate = substrate or get_substrate()
         self.entity_id = entity_id
         self.model = model or get_current_model()
 
-        if substrate == 'k8s':
+        if substrate == "k8s":
             self.connector = K8sConnector()
 
-        elif substrate == 'machine':
+        elif substrate == "machine":
             self.connector = MachineConnector()
 
         else:
diff --git a/jhack/mongo/query_k8s_controller.sh b/jhack/mongo/query_k8s_controller.sh
index 0d970d0..5eaf58b 100644
--- a/jhack/mongo/query_k8s_controller.sh
+++ b/jhack/mongo/query_k8s_controller.sh
@@ -1,8 +1,12 @@
 #!/bin/bash
+set -x
 kctl=${1}
 user=${2}
 password=${3}
 k8s_ns=${4}
 k8s_controller_pod=${5}
 query=${6}
-${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'"
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --ssl --sslAllowInvalidCertificates --username '${user}' --password '${password}' --help"
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'"
+microk8s.kubectl exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password} --eval ${query}"
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password}"
diff --git a/jhack/mongo/query_machine_controller.sh b/jhack/mongo/query_machine_controller.sh
index 15f7684..42a94ed 100644
--- a/jhack/mongo/query_machine_controller.sh
+++ b/jhack/mongo/query_machine_controller.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
+set -x
 client=${1}
 user=${2}
 password=${3}
-controller=${4}
+model=${4}
 machine=${5}
 query=${6}
-juju ssh -m "$controller" "$machine" -- "$client" '127.0.0.1:37017/juju' --authenticationDatabase admin --tls --tlsAllowInvalidCertificates --quiet --username "$user" --password "$password" --eval "$query"
+juju ssh -m "$model" "$machine" -- "$client" '127.0.0.1:37017/juju' --authenticationDatabase admin --tls --tlsAllowInvalidCertificates --quiet --username "$user" --password "$password" --eval "$query"
diff --git a/jhack/tests/config/test_config.py b/jhack/tests/config/test_config.py
index 20cb6a2..6ac1653 100644
--- a/jhack/tests/config/test_config.py
+++ b/jhack/tests/config/test_config.py
@@ -64,4 +64,3 @@ def test_defaults():
         assert cfg.get("test", "bar") == "baz"
 
     Config._DEFAULTS = old_def
-
diff --git a/jhack/tests/mongo/test_k8s_connector_manual.py b/jhack/tests/mongo/test_k8s_connector_manual.py
index e8db225..8d902c6 100644
--- a/jhack/tests/mongo/test_k8s_connector_manual.py
+++ b/jhack/tests/mongo/test_k8s_connector_manual.py
@@ -1,8 +1,15 @@
 from jhack.mongo.mongo import K8sConnector
 
 
-def test_k8s_connector():
+def test_k8s_connector_base():
     connector = K8sConnector()
-    query = 'db.relations.find({"key": "loki:logging"}).pretty()'
+    query = r"db.relations.find()"
+    val = connector.query(query)
+    assert val
+
+
+def test_k8s_connector_relation():
+    connector = K8sConnector()
+    query = r'db.relations.find({"key": "grafana:catalogue catalogue:catalogue"})'
     val = connector.query(query)
     assert val
diff --git a/jhack/tests/mongo/test_machine_connector_manual.py b/jhack/tests/mongo/test_machine_connector_manual.py
index 109b784..33f8476 100644
--- a/jhack/tests/mongo/test_machine_connector_manual.py
+++ b/jhack/tests/mongo/test_machine_connector_manual.py
@@ -1,8 +1,15 @@
 from jhack.mongo.mongo import MachineConnector
 
 
-def test_k8s_connector():
-    connector = MachineConnector()
+def test_machine_connector_base():
+    connector = MachineConnector("lxdcloud")
+    query = r"db.relations.find()"
+    val = connector.query(query)
+    assert val
+
+
+def test_machine_connector():
+    connector = MachineConnector("lxdcloud")
     query = 'db.relations.find({"key": "kafka:cluster"})'
     val = connector.query(query)
     assert val

From cdedd5c46d528267056a033ba6e2808704dbe1a7 Mon Sep 17 00:00:00 2001
From: Pietro Pasotti <starfire.daemon@gmail.com>
Date: Fri, 5 May 2023 14:57:19 +0200
Subject: [PATCH 3/5] all green

---
 jhack/mongo/mongo.py                          | 57 +++++++++++++------
 .../tests/mongo/test_k8s_connector_manual.py  |  4 +-
 .../mongo/test_machine_connector_manual.py    |  4 +-
 3 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py
index c406974..9f0a8a8 100644
--- a/jhack/mongo/mongo.py
+++ b/jhack/mongo/mongo.py
@@ -5,7 +5,7 @@
 import shlex
 from pathlib import Path
 from subprocess import PIPE, Popen
-from typing import Literal, Tuple
+from typing import Literal, Tuple, List
 
 from jhack.helpers import JPopen, get_current_model, get_substrate
 
@@ -28,6 +28,14 @@ def to_json(query_result: str):
     return json.loads(jsn_str)
 
 
+class TooManyResults(RuntimeError):
+    """Raised when a query returns more results than we handle."""
+
+
+class EmptyQueryResult(RuntimeError):
+    """Query returned no results."""
+
+
 class ConnectorBase:
     args_getter_script: Path
     query_script: Path
@@ -58,35 +66,48 @@ def get_args(self) -> Tuple[str, ...]:
         return tuple(out.split())  # noqa
         # return tuple(f"'{x}'" for x in out.split())  # noqa
 
-    def query(self, query: str):
+    def get_one(self, query: str) -> dict:
+        return self.get_many(query, raise_if_too_many=False)[0]
+
+    def get_many(self, query: str, raise_if_too_many=True) -> List[dict]:
         if "pretty()" in query:
             # we need one result per line to be able to deserialize.
             raise ValueError("invalid query: unsupported pretty() statement")
         raw = self._escape_query(query)
-        return self._run_query(raw)
+        return self._run_query(raw, raise_if_too_many=raise_if_too_many)
 
-    def _run_query(self, query: str):
+    def _run_query(self, query: str, raise_if_too_many=True):
         command = ["bash", str(self.query_script.absolute()), *self.args, query]
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
-        out = proc.stdout.read().decode("utf-8")
-        if not out:
+        raw_output = proc.stdout.read().decode("utf-8")
+        if not raw_output:
             err = proc.stderr.read().decode("utf-8")
             print(err)
             raise RuntimeError(f"unexpected result from command {command}; {err!r}")
 
-        txt = out.split("\n")
-        # todo: some queries return a list of documents.
-        result = txt[1]
-
-        try:
-            return to_json(result)
-        except Exception as e:
+        txt = raw_output.split("\n")
+        out = []
+        for value in txt[1:]:
+            if not value:
+                continue
+            if value == 'Type "it" for more':
+                if raise_if_too_many:
+                    raise TooManyResults()
+                continue
+            try:
+                out.append(to_json(value))
+            except Exception as e:
+                err = proc.stderr.read().decode("utf-8")
+                print(err)
+                raise RuntimeError(
+                    f"failed deserializing query result {value} with {type(e)} {err}"
+                ) from e
+        if not out:
             err = proc.stderr.read().decode("utf-8")
             print(err)
-            raise RuntimeError(
-                f"failed deserializing query result {result} with {type(e)} {err}"
-            ) from e
+            raise EmptyQueryResult()
 
+        return out
 
 class K8sConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
@@ -129,5 +150,5 @@ def __init__(
         else:
             raise TypeError(substrate)
 
-    def query(self, q: str):
-        return self.connector.query(q)
+    def get_one(self, q: str):
+        return self.connector.get_one(q)
diff --git a/jhack/tests/mongo/test_k8s_connector_manual.py b/jhack/tests/mongo/test_k8s_connector_manual.py
index 8d902c6..0d51bc2 100644
--- a/jhack/tests/mongo/test_k8s_connector_manual.py
+++ b/jhack/tests/mongo/test_k8s_connector_manual.py
@@ -4,12 +4,12 @@
 def test_k8s_connector_base():
     connector = K8sConnector()
     query = r"db.relations.find()"
-    val = connector.query(query)
+    val = connector.get_one(query)
     assert val
 
 
 def test_k8s_connector_relation():
     connector = K8sConnector()
     query = r'db.relations.find({"key": "grafana:catalogue catalogue:catalogue"})'
-    val = connector.query(query)
+    val = connector.get_one(query)
     assert val
diff --git a/jhack/tests/mongo/test_machine_connector_manual.py b/jhack/tests/mongo/test_machine_connector_manual.py
index 33f8476..18f91e0 100644
--- a/jhack/tests/mongo/test_machine_connector_manual.py
+++ b/jhack/tests/mongo/test_machine_connector_manual.py
@@ -4,12 +4,12 @@
 def test_machine_connector_base():
     connector = MachineConnector("lxdcloud")
     query = r"db.relations.find()"
-    val = connector.query(query)
+    val = connector.get_many(query)
     assert val
 
 
 def test_machine_connector():
     connector = MachineConnector("lxdcloud")
     query = 'db.relations.find({"key": "kafka:cluster"})'
-    val = connector.query(query)
+    val = connector.get_one(query)
     assert val

From 20e1fcd32590c94b3337089554262a80e85d58ec Mon Sep 17 00:00:00 2001
From: Pietro Pasotti <starfire.daemon@gmail.com>
Date: Fri, 5 May 2023 14:57:26 +0200
Subject: [PATCH 4/5] lint

---
 jhack/mongo/mongo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py
index 9f0a8a8..b119974 100644
--- a/jhack/mongo/mongo.py
+++ b/jhack/mongo/mongo.py
@@ -5,7 +5,7 @@
 import shlex
 from pathlib import Path
 from subprocess import PIPE, Popen
-from typing import Literal, Tuple, List
+from typing import List, Literal, Tuple
 
 from jhack.helpers import JPopen, get_current_model, get_substrate
 
@@ -109,6 +109,7 @@ def _run_query(self, query: str, raise_if_too_many=True):
 
         return out
 
+
 class K8sConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
 

From e6856254f1294650a347fb574769e4a94ce3f95a Mon Sep 17 00:00:00 2001
From: Pietro Pasotti <starfire.daemon@gmail.com>
Date: Wed, 10 May 2023 12:00:35 +0200
Subject: [PATCH 5/5] mongo progress

---
 jhack/mongo/mongo.py                          | 68 ++++++++-----------
 jhack/mongo/query_k8s_controller.sh           |  2 +-
 .../tests/mongo/test_k8s_connector_manual.py  | 19 ++++--
 .../mongo/test_machine_connector_manual.py    | 12 ++--
 4 files changed, 50 insertions(+), 51 deletions(-)

diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py
index b119974..47717e6 100644
--- a/jhack/mongo/mongo.py
+++ b/jhack/mongo/mongo.py
@@ -66,17 +66,20 @@ def get_args(self) -> Tuple[str, ...]:
         return tuple(out.split())  # noqa
         # return tuple(f"'{x}'" for x in out.split())  # noqa
 
-    def get_one(self, query: str) -> dict:
-        return self.get_many(query, raise_if_too_many=False)[0]
+    def get(self, query: str, n: int = None, query_filter: str = None) -> List[dict]:
+        qfilter = query_filter or "{}"
+        if n is None:
+            q = query + fr".find({qfilter}).toArray()"
+        else:
+            q = query + fr".find({qfilter}).limit({n}).toArray()"
+        escaped = self._escape_query(q)
+        out = self._run_query(escaped)
 
-    def get_many(self, query: str, raise_if_too_many=True) -> List[dict]:
-        if "pretty()" in query:
-            # we need one result per line to be able to deserialize.
-            raise ValueError("invalid query: unsupported pretty() statement")
-        raw = self._escape_query(query)
-        return self._run_query(raw, raise_if_too_many=raise_if_too_many)
+        if not out:
+            raise EmptyQueryResult(escaped)
+        return out
 
-    def _run_query(self, query: str, raise_if_too_many=True):
+    def _run_query(self, query: str):
         command = ["bash", str(self.query_script.absolute()), *self.args, query]
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
         raw_output = proc.stdout.read().decode("utf-8")
@@ -85,36 +88,23 @@ def _run_query(self, query: str, raise_if_too_many=True):
             print(err)
             raise RuntimeError(f"unexpected result from command {command}; {err!r}")
 
-        txt = raw_output.split("\n")
-        out = []
-        for value in txt[1:]:
-            if not value:
-                continue
-            if value == 'Type "it" for more':
-                if raise_if_too_many:
-                    raise TooManyResults()
-                continue
-            try:
-                out.append(to_json(value))
-            except Exception as e:
-                err = proc.stderr.read().decode("utf-8")
-                print(err)
-                raise RuntimeError(
-                    f"failed deserializing query result {value} with {type(e)} {err}"
-                ) from e
-        if not out:
+        # stripped = "[" + "\n".join(filter(None, raw_output.split('\n')[1:])) + "]"
+        stripped = "\n".join(filter(None, raw_output.split('\n')[1:]))
+        try:
+            return to_json(stripped)
+        except Exception as e:
             err = proc.stderr.read().decode("utf-8")
             print(err)
-            raise EmptyQueryResult()
-
-        return out
+            raise RuntimeError(
+                f"failed deserializing query result {stripped} with {type(e)} {err}"
+            ) from e
 
 
 class K8sConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
 
     args_getter_script = (
-        Path(__file__).parent / "get_credentials_from_k8s_controller.sh"
+            Path(__file__).parent / "get_credentials_from_k8s_controller.sh"
     )
     query_script = Path(__file__).parent / "query_k8s_controller.sh"
 
@@ -123,7 +113,7 @@ class MachineConnector(ConnectorBase):
     """Mongo database connector for kubernetes controllers."""
 
     args_getter_script = (
-        Path(__file__).parent / "get_credentials_from_machine_controller.sh"
+            Path(__file__).parent / "get_credentials_from_machine_controller.sh"
     )
     query_script = Path(__file__).parent / "query_machine_controller.sh"
 
@@ -133,10 +123,10 @@ def get_args(self):
 
 class Mongo:
     def __init__(
-        self,
-        entity_id: int = 0,
-        substrate: Literal["k8s", "machine"] = None,
-        model: str = None,
+            self,
+            entity_id: int = 0,
+            substrate: Literal["k8s", "machine"] = None,
+            model: str = None,
     ):
         self.substrate = substrate or get_substrate()
         self.entity_id = entity_id
@@ -151,5 +141,7 @@ def __init__(
         else:
             raise TypeError(substrate)
 
-    def get_one(self, q: str):
-        return self.connector.get_one(q)
+    def _get(self, query: str, n: int = None, query_filter: str = None):
+        return self.connector.get(query, n=n, query_filter=query_filter)
+
+
diff --git a/jhack/mongo/query_k8s_controller.sh b/jhack/mongo/query_k8s_controller.sh
index 5eaf58b..056b211 100644
--- a/jhack/mongo/query_k8s_controller.sh
+++ b/jhack/mongo/query_k8s_controller.sh
@@ -8,5 +8,5 @@ k8s_controller_pod=${5}
 query=${6}
 #${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --ssl --sslAllowInvalidCertificates --username '${user}' --password '${password}' --help"
 #${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'"
-microk8s.kubectl exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password} --eval ${query}"
+microk8s.kubectl exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -t -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password} --eval ${query}"
 #${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password}"
diff --git a/jhack/tests/mongo/test_k8s_connector_manual.py b/jhack/tests/mongo/test_k8s_connector_manual.py
index 0d51bc2..00da8ca 100644
--- a/jhack/tests/mongo/test_k8s_connector_manual.py
+++ b/jhack/tests/mongo/test_k8s_connector_manual.py
@@ -3,13 +3,20 @@
 
 def test_k8s_connector_base():
     connector = K8sConnector()
-    query = r"db.relations.find()"
-    val = connector.get_one(query)
-    assert val
+    query = r"db.relations"
+    val = connector.get(query, n=1)
+    assert len(val) == 1
+
+
+def test_k8s_connector_get_all():
+    connector = K8sConnector()
+    query = r"db.relations"
+    val = connector.get(query)
+    assert len(val) == 26
 
 
 def test_k8s_connector_relation():
     connector = K8sConnector()
-    query = r'db.relations.find({"key": "grafana:catalogue catalogue:catalogue"})'
-    val = connector.get_one(query)
-    assert val
+    query = r'db.relations'
+    val = connector.get(query, query_filter='{"key": "grafana:catalogue catalogue:catalogue"}')
+    assert len(val) == 1
\ No newline at end of file
diff --git a/jhack/tests/mongo/test_machine_connector_manual.py b/jhack/tests/mongo/test_machine_connector_manual.py
index 18f91e0..eec9e15 100644
--- a/jhack/tests/mongo/test_machine_connector_manual.py
+++ b/jhack/tests/mongo/test_machine_connector_manual.py
@@ -3,13 +3,13 @@
 
 def test_machine_connector_base():
     connector = MachineConnector("lxdcloud")
-    query = r"db.relations.find()"
-    val = connector.get_many(query)
-    assert val
+    query = r"db.relations"
+    val = connector.get(query, n=1)
+    assert len(val) == 1
 
 
 def test_machine_connector():
     connector = MachineConnector("lxdcloud")
-    query = 'db.relations.find({"key": "kafka:cluster"})'
-    val = connector.get_one(query)
-    assert val
+    query = 'db.relations'
+    val = connector.get(query, query_filter='{"key": "kafka:cluster"}')
+    assert len(val) == 1