juju controller mongodb hackery [work in progress] [do not merge]

jhack/conf/conf.py

@@ -58,10 +58,12 @@ def get(self, *path: str) -> bool:  # todo: add more toml types?
                 data = data[item]
             except KeyError:
                 if self._path is self._DEFAULTS:
-                    logger.error(f'{item} not found in default config; invalid path')
+                    logger.error(f"{item} not found in default config; invalid path")
                     raise
 
-                logger.info(f'{item} not found in user-config {self._path}; defaulting...')
+                logger.info(
+                    f"{item} not found in user-config {self._path}; defaulting..."
+                )
                 return self.get_default(*path)
         return data
 

jhack/helpers.py

@@ -39,6 +39,14 @@ def check_command_available(cmd: str):
     return proc.returncode == 0
 
 
+def get_current_controller() -> str:
+    cmd = f"juju whoami --format=json"
+    proc = JPopen(cmd.split())
+    raw = proc.stdout.read().decode("utf-8")
+    whoami_info = jsn.loads(raw)
+    return whoami_info["controller"]
+
+
 def get_substrate(model: str = None) -> Literal["k8s", "machine"]:
     """Attempts to guess whether we're talking k8s or machine."""
     cmd = f'juju show-model{f" {model}" if model else ""} --format=json'

jhack/mongo/eson.py

@@ -0,0 +1,54 @@
+def emit_thing(v):
+    if v.__class__.__name__ == "dict":
+        return emit_dict(v)
+    elif v.__class__.__name__ == "list":
+        return emit_list(v)
+    elif v.__class__.__name__ in {
+        "Int64",
+        "int",
+        "long",
+        "float",
+        "decimal",
+        "Decimal128",
+        "Decimal",
+    }:
+        return str(v)
+    elif v.__class__.__name__ == "datetime":
+        return v
+    else:
+        return str(v)
+
+
+def emit_list(ll: list) -> list:
+    return list(map(emit_thing, ll))
+
+
+def emit_dict(dd: dict) -> dict:
+    out = {}
+    for k, v in dd.items():
+        out[k] = emit_thing(v)
+    return out
+
+
+def parse_eson(doc: str) -> dict:
+    return emit_dict(doc)
+
+
+import json
+import re
+
+from bson import json_util
+
+
+def read_mongoextjson_file(filename):
+    with open(filename, "r") as f:
+        bsondata = f.read()
+        # Convert Mongo object(s) to regular strict JSON
+        jsondata = re.sub(
+            r"ObjectId\s*\(\s*\"(\S+)\"\s*\)", r'{"$oid": "\1"}', bsondata
+        )
+        # Description of Mongo ObjectId:
+        # https://docs.mongodb.com/manual/reference/mongodb-extended-json/#mongodb-bsontype-ObjectId
+        # now we can parse this as JSON, and use MongoDB's object_hook
+        data = json.loads(jsondata, object_hook=json_util.object_hook)
+        return data

jhack/mongo/get_credentials_from_k8s_controller.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+kubectl_bin=/snap/bin/microk8s.kubectl
+k8s_ns=$(juju whoami | grep Controller | awk '{print "controller-"$2}')
+k8s_controller_pod=$(${kubectl_bin} -n "${k8s_ns}" get pods | grep -E "^controller-([0-9]+)" | awk '{print $1}')
+mongo_user=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep tag /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'")
+mongo_pass=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep statepassword /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'")
+
+echo "$kubectl_bin" "$mongo_user" "$mongo_pass" "$k8s_ns" "$k8s_controller_pod"

jhack/mongo/get_credentials_from_machine_controller.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+machine=${1}
+model=${2}
+
+read -d '' -r cmds <<'EOF'
+conf=/var/lib/juju/agents/machine-*/agent.conf
+user=`sudo grep tag $conf | cut -d' ' -f2`
+password=`sudo grep statepassword $conf | cut -d' ' -f2`
+if [ -f /snap/bin/juju-db.mongo ]; then
+  client=/snap/bin/juju-db.mongo
+elif [ -f /usr/lib/juju/mongo*/bin/mongo ]; then
+  client=/usr/lib/juju/mongo*/bin/mongo
+else
+  client=/usr/bin/mongo
+fi
+echo "$client" "$user" "$password"
+EOF
+juju ssh -m "${model}" "${machine}" "${cmds}"

jhack/mongo/mongo.py

@@ -0,0 +1,147 @@
+#!/bin/bash
+import json
+import os
+import re
+import shlex
+from pathlib import Path
+from subprocess import PIPE, Popen
+from typing import List, Literal, Tuple
+
+from jhack.helpers import JPopen, get_current_model, get_substrate
+
+
+def escape_double_quotes(query):
+    return query.replace('"', r"\"")
+
+
+def numberlong(s: str):
+    return re.sub(r"NumberLong\((\d+)\)", r"\1", s)
+
+
+FILTERS = [numberlong]
+
+
+def to_json(query_result: str):
+    jsn_str = query_result
+    for f in FILTERS:
+        jsn_str = f(jsn_str)
+    return json.loads(jsn_str)
+
+
+class TooManyResults(RuntimeError):
+    """Raised when a query returns more results than we handle."""
+
+
+class EmptyQueryResult(RuntimeError):
+    """Query returned no results."""
+
+
+class ConnectorBase:
+    args_getter_script: Path
+    query_script: Path
+
+    def __init__(self, controller: str = None, unit_id: int = 0):
+        self.controller = controller
+        self.model = f"{controller}:controller" if controller else "controller"
+        self.unit_id = unit_id
+        self.args = self.get_args()
+
+    def _escape_query(self, query: str) -> str:
+        return rf'"{escape_double_quotes(query)}"'
+
+    def _get_output(self, cmd: str) -> str:
+        return JPopen(shlex.split(cmd)).stdout.read().decode("utf-8")
+
+    def get_args(self) -> Tuple[str, ...]:
+        out = (
+            Popen(
+                shlex.split(
+                    f"bash {self.args_getter_script.absolute()} {self.unit_id} {self.model}",
+                ),
+                stdout=PIPE,
+            )
+            .stdout.read()
+            .decode("utf-8")
+        )
+        return tuple(out.split())  # noqa
+        # return tuple(f"'{x}'" for x in out.split())  # noqa
+
+    def get(self, query: str, n: int = None, query_filter: str = None) -> List[dict]:
+        qfilter = query_filter or "{}"
+        if n is None:
+            q = query + fr".find({qfilter}).toArray()"
+        else:
+            q = query + fr".find({qfilter}).limit({n}).toArray()"
+        escaped = self._escape_query(q)
+        out = self._run_query(escaped)
+
+        if not out:
+            raise EmptyQueryResult(escaped)
+        return out
+
+    def _run_query(self, query: str):
+        command = ["bash", str(self.query_script.absolute()), *self.args, query]
+        proc = Popen(command, stdout=PIPE, stderr=PIPE)
+        raw_output = proc.stdout.read().decode("utf-8")
+        if not raw_output:
+            err = proc.stderr.read().decode("utf-8")
+            print(err)
+            raise RuntimeError(f"unexpected result from command {command}; {err!r}")
+
+        # stripped = "[" + "\n".join(filter(None, raw_output.split('\n')[1:])) + "]"
+        stripped = "\n".join(filter(None, raw_output.split('\n')[1:]))
+        try:
+            return to_json(stripped)
+        except Exception as e:
+            err = proc.stderr.read().decode("utf-8")
+            print(err)
+            raise RuntimeError(
+                f"failed deserializing query result {stripped} with {type(e)} {err}"
+            ) from e
+
+
+class K8sConnector(ConnectorBase):
+    """Mongo database connector for kubernetes controllers."""
+
+    args_getter_script = (
+            Path(__file__).parent / "get_credentials_from_k8s_controller.sh"
+    )
+    query_script = Path(__file__).parent / "query_k8s_controller.sh"
+
+
+class MachineConnector(ConnectorBase):
+    """Mongo database connector for kubernetes controllers."""
+
+    args_getter_script = (
+            Path(__file__).parent / "get_credentials_from_machine_controller.sh"
+    )
+    query_script = Path(__file__).parent / "query_machine_controller.sh"
+
+    def get_args(self):
+        return super().get_args() + (self.model, str(self.unit_id))
+
+
+class Mongo:
+    def __init__(
+            self,
+            entity_id: int = 0,
+            substrate: Literal["k8s", "machine"] = None,
+            model: str = None,
+    ):
+        self.substrate = substrate or get_substrate()
+        self.entity_id = entity_id
+        self.model = model or get_current_model()
+
+        if substrate == "k8s":
+            self.connector = K8sConnector()
+
+        elif substrate == "machine":
+            self.connector = MachineConnector()
+
+        else:
+            raise TypeError(substrate)
+
+    def _get(self, query: str, n: int = None, query_filter: str = None):
+        return self.connector.get(query, n=n, query_filter=query_filter)
+
+

jhack/mongo/query_k8s_controller.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -x
+kctl=${1}
+user=${2}
+password=${3}
+k8s_ns=${4}
+k8s_controller_pod=${5}
+query=${6}
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --ssl --sslAllowInvalidCertificates --username '${user}' --password '${password}' --help"
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'"
+microk8s.kubectl exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -t -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password} --eval ${query}"
+#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password}"

jhack/mongo/query_machine_controller.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+client=${1}
+user=${2}
+password=${3}
+model=${4}
+machine=${5}
+query=${6}
+juju ssh -m "$model" "$machine" -- "$client" '127.0.0.1:37017/juju' --authenticationDatabase admin --tls --tlsAllowInvalidCertificates --quiet --username "$user" --password "$password" --eval "$query"

jhack/tests/config/test_config.py

@@ -64,4 +64,3 @@ def test_defaults():
         assert cfg.get("test", "bar") == "baz"
 
     Config._DEFAULTS = old_def
-

jhack/tests/mongo/test_k8s_connector_manual.py

@@ -0,0 +1,22 @@
+from jhack.mongo.mongo import K8sConnector
+
+
+def test_k8s_connector_base():
+    connector = K8sConnector()
+    query = r"db.relations"
+    val = connector.get(query, n=1)
+    assert len(val) == 1
+
+
+def test_k8s_connector_get_all():
+    connector = K8sConnector()
+    query = r"db.relations"
+    val = connector.get(query)
+    assert len(val) == 26
+
+
+def test_k8s_connector_relation():
+    connector = K8sConnector()
+    query = r'db.relations'
+    val = connector.get(query, query_filter='{"key": "grafana:catalogue catalogue:catalogue"}')
+    assert len(val) == 1

jhack/tests/mongo/test_machine_connector_manual.py

@@ -0,0 +1,15 @@
+from jhack.mongo.mongo import MachineConnector
+
+
+def test_machine_connector_base():
+    connector = MachineConnector("lxdcloud")
+    query = r"db.relations"
+    val = connector.get(query, n=1)
+    assert len(val) == 1
+
+
+def test_machine_connector():
+    connector = MachineConnector("lxdcloud")
+    query = 'db.relations'
+    val = connector.get(query, query_filter='{"key": "kafka:cluster"}')
+    assert len(val) == 1