Bash *? wildcard support for remote resource export/import

README.md

@@ -134,6 +134,10 @@ history = ~/.grafcli_history
 verbose = off
 # Answer 'yes' to all overwrite prompts.
 force = on
+# Forcibly nullify the dashboard ID when importing a dashboard
+# that was exported from some other host. Enable this if you
+# wish to move dashboards between hosts using export/import.
+nullify_dbid = off
 
 [resources]
 # Directory where all local data will be stored (including backups).
@@ -306,3 +310,53 @@ Some of the common operations.
 ```
 [/]> export templates/dashboards/dashboard ~/dashboard.json
 ```
+
+## Basic Wildcard Support for Remote Resources
+
+Caveats/Notes
+* The following is only supported when the `export` source or or `import` destination is a `remote` resource.
+* The grafcli `ls` command is not able to navigate the `exports` hierarchy beyond the first level.
+* All wildcards operate on the doc name/dashboard name (aka slug) not the actual filename, hence the .json part is not considered
+
+The `export` and `import` command have support for Bash style wildcards (e.g. ? and *) within the names of dashboards to export and import. The command be run from the CLI or from batch mode (take care when passing absolute paths with * in them from the shell, you may need to use "" around the path)
+
+The `export` command may be used to bulk export every dashboard or a selection of dashboards from some host xyz, and then place them either into a locally specified directory, an absolute directory path, or a newly created subdirectory of the `data-dir` named `exports/xyz`. Example usage follows:
+
+* Export all dashboards from host xyz, placing them into ./xyz_exports
+
+```
+[/]> export remote/xyz/* xyz_exports
+```
+
+* Export all dashboards matching the name `db*exporter*` from host xyz, placing them into `/home/someone/somewhere/exports`
+
+```
+[/]> export remote/xyz/*exporter* /home/someone/somewhere/exports
+```
+
+* Export all dashboards matching the name `thing?` from host xyz, (auto)placing them into `<data-dir>/exports/xyz` (this is a convenience shortcut)
+
+```
+[/]> export remote/xyz/thing?
+```
+
+The `import` command supports the reverse operation, where we may wish to import (upload) a selection of dashboards to some other host. In this case, it is essential that the configuration item `nullify_dbid` is set to `on` in order to prevent errors when importing.
+
+* Import all dashboards previously exported from host xyz, to host abc.
+
+```
+[/]> import exports/xyz/* remote/abc
+```
+
+* Import all dashboards matching the name `*hdfs*` previously exported from host xyz, to host abc.
+
+```
+[/]> import exports/xyz/*hdfs* remote/abc
+```
+
+* Import all dashboards matching the name `thing-?` from a direct ~ location, to host abc
+
+```
+[/]> import ~/dashboards/production/thing-? remote/abc
+```
+

grafcli.conf.example

@@ -4,6 +4,7 @@ history = ~/.grafcli_history
 verbose = off
 force = on
 colorize = on
+nullify_dbid = off
 
 [resources]
 data-dir = ~/.grafcli

grafcli/commands.py

@@ -1,3 +1,4 @@
+import fnmatch
 import os
 import re
 import json
@@ -7,7 +8,7 @@
 from climb.config import config
 from climb.commands import Commands, command, completers
 from climb.exceptions import CLIException
-from climb.paths import format_path, split_path, ROOT_PATH
+from climb.paths import format_path, split_path, ROOT_PATH, SEPARATOR
 
 from grafcli.documents import Document, Dashboard, Row, Panel
 from grafcli.exceptions import CommandCancelled
@@ -208,21 +209,115 @@ def restore(self, system_path, path):
 
         shutil.rmtree(tmp_dir)
 
+    def file_export_all(self, path_wildcard, system_path=None):
+        """ Perform a bulk export 
+            - path_wildcard is typically going to be of the form 
+              remote/abc/* or remote/abc/xyz*dash*
+              where Unix filename matching is performed against the doc_name
+            - system_path may be of the form of a local directory
+              name, an absolute path to a director, or, if ommitted
+              data-dir/exports/abc will be created and used.
+        """
+        path_parts = split_path(path_wildcard)
+        if len(path_parts) != 3:
+            raise CLIException("Export all needs a host (wildcard hosts not yet supported)")
+
+        location = path_parts[0]
+        host = path_parts[1]
+        wildcard = path_parts[2]
+
+        if location != "remote":
+            raise CLIException("Export all is only available for remote resources")
+
+        # Is this shortcut desirable?
+        if not system_path:
+            system_path = self._resources.local_system_path("exports", host)
+
+        os.makedirs(system_path, exist_ok=True)
+        documents = self._resources.list(SEPARATOR.join(path_parts[0:2]))
+        for doc_name in documents:            
+            if not fnmatch.fnmatch(doc_name, wildcard):
+                continue
+            file_path = os.path.join(system_path, to_file_format(doc_name))
+            get_path = os.path.join(location, host, doc_name)
+            doc = self._resources.get(get_path)
+            with open(file_path, 'w') as file:
+                file.write(json_pretty(doc.source))
+            self._cli.log("export: {} -> {}", get_path, file_path)
+
     @command
     @completers('path', 'system_path')
     def file_export(self, path, system_path):
+        if not path:
+            raise CLIException("No path provided")
+
+        parts = split_path(path)
+        if "*" in parts[-1] or "?" in parts[-1]:
+            self.file_export_all(path, system_path)
+            return
+
+        if not system_path:
+            raise CLIException("No system path provided")
+
         path = format_path(self._cli.current_path, path)
         system_path = os.path.expanduser(system_path)
         document = self._resources.get(path)
-
         with open(system_path, 'w') as file:
             file.write(json_pretty(document.source))
 
         self._cli.log("export: {} -> {}", path, system_path)
 
+    def file_import_all(self, system_path_wildcard, path):
+        """ Perform a bulk import
+            - system_path_wildcard may be of the form 
+              /home/abc/exports/xyz/* or exports/xyz/*things*
+              where Unix filename matching is performed on the dashboard name
+            - path is typically going to be of the form
+              remote/abc
+        """
+
+        # don't use split_path for system_path_wildcard to preserve any leading /
+        system_parts = system_path_wildcard.split(SEPARATOR)
+        path_parts = split_path(path)
+        if len(path_parts) != 2:
+            raise CLIException("Import all needs a host (wildcard hosts not yet supported)")
+
+        wildcard = system_parts[-1]
+        system_path = SEPARATOR.join(system_parts[0:-1])
+        location = path_parts[0]
+
+        if location != "remote":
+            raise CLIException("Import all is only available for remote resources")
+
+        abs_system_path = os.path.expanduser(system_path)
+
+        # check if it's an absolute path, if not check shortcut for 'exports'
+        if not os.path.isdir(abs_system_path):
+            abs_system_path = self._resources.local_system_path(path=system_path)
+            if not os.path.isdir(abs_system_path):
+                raise CLIException("Cannot find source path {}".format(system_path_wildcard))
+
+        dst_path = format_path(self._cli.current_path, path)
+        dashboards = self._resources.list_path(abs_system_path)
+        for dashboard_name in dashboards:
+            if not fnmatch.fnmatch(dashboard_name, wildcard):
+                continue
+
+            file_path = os.path.join(abs_system_path, to_file_format(dashboard_name))
+            with open(file_path, 'r') as file:
+                content = file.read()
+            document = Document.from_source(json.loads(content))
+            self._resources.save(dst_path, document)
+            self._cli.log("import: {} -> {}/{}", file_path, dst_path, dashboard_name)
+
     @command
     @completers('system_path', 'path')
     def file_import(self, system_path, path, match_slug=False):
+        parts = split_path(system_path)
+        if "*" in parts[-1] or "?" in parts[-1]:
+            self.file_import_all(system_path, path)
+            return
+
         system_path = os.path.expanduser(system_path)
         path = format_path(self._cli.current_path, path)
 

grafcli/resources/local.py

@@ -4,4 +4,8 @@
 
 class LocalResources(CommonResources):
     def __init__(self, local_dir):
+        self.local_dir = local_dir
         self._storage = SystemStorage(local_dir)
+
+    def list_path(self, path):
+        return self._storage.list(path)

grafcli/resources/resources.py

@@ -1,3 +1,4 @@
+import os
 from climb.config import config
 from climb.paths import split_path
 
@@ -6,14 +7,15 @@
 from grafcli.resources.templates import DashboardsTemplates, RowsTemplates, PanelTemplates, CATEGORIES
 from grafcli.resources.local import LocalResources
 
-LOCAL_DIR = 'backups'
-
+BACKUP_DIR = 'backups'
+EXPORT_DIR = 'exports'
 
 class Resources(object):
 
     def __init__(self):
         self._resources = {
-            'backups': LocalResources(LOCAL_DIR),
+            'backups': LocalResources(BACKUP_DIR),
+            'exports': LocalResources(EXPORT_DIR),
             'remote': {},
             'templates': {
                 'dashboards': DashboardsTemplates(),
@@ -22,6 +24,21 @@ def __init__(self):
             },
         }
 
+    def local_system_path(self, resource=None, path=None):
+        """ Lookup the expanded local system path for some
+            resource and/or path
+        """
+        p = os.path.join(config['resources']['data-dir'])
+        if resource:
+            if resource not in self._resources:
+                raise InvalidPath("Invalid resource {}".format(resource))
+            p = os.path.join(config['resources']['data-dir'],
+                             self._resources[resource].local_dir)
+        if path is not None:
+            p = os.path.join(p, path)
+
+        return os.path.expanduser(p)
+
     def list(self, path):
         """Returns list of sub-nodes for given path."""
         try:
@@ -37,6 +54,12 @@ def list(self, path):
 
         return manager.list(*parts)
 
+    def list_path(self, path):
+        """Use LocalResources to list files 
+           in a SystemStorage path
+        """
+        return LocalResources("").list_path(path)
+
     def get(self, path):
         """Returns resource data."""
         manager, parts = self._parse_path(path)

grafcli/storage/api.py

@@ -53,6 +53,12 @@ def save(self, dashboard_id, dashboard):
                 raise
             data["dashboard"]["id"] = None
 
+        if config['grafcli'].getboolean("nullify_dbid"):
+            # No matter whether the db exists, nullify the ID to prevent 
+            # 404 not found when moving a dashboard from some host xyz 
+            # to host abc (else the below post will 404)
+            data["dashboard"]["id"] = None
+
         self._call('POST', 'dashboards/db', data)
 
     def remove(self, dashboard_id):

grafcli/storage/system.py

@@ -15,8 +15,11 @@ def __init__(self, base_dir):
         self._base_dir = base_dir
         makepath(self._base_dir)
 
-    def list(self):
-        return list_files(self._base_dir)
+    def list(self, path=None):
+        list_path = self._base_dir
+        if path:
+            list_path = os.path.join(self._base_dir, path)
+        return list_files(list_path)
 
     def get(self, dashboard_id):
         try:

tests/test_resources.py

@@ -30,7 +30,7 @@ def tearDown(self):
     def test_list(self):
         r = Resources()
 
-        self.assertEqual(r.list(None), ['backups', 'remote', 'templates'])
+        self.assertEqual(r.list(None), ['backups', 'exports', 'remote', 'templates'])
         self.assertEqual(r.list('remote'), ['localhost'])
         self.assertEqual(r.list('templates'), ('dashboards', 'rows', 'panels'))