diff --git a/README.md b/README.md index 12f6941..8f7b31c 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ history = ~/.grafcli_history verbose = off # Answer 'yes' to all overwrite prompts. force = on +# Forcibly nullify the dashboard ID when importing a dashboard +# that was exported from some other host. Enable this if you +# wish to move dashboards between hosts using export/import. +nullify_dbid = off [resources] # Directory where all local data will be stored (including backups). @@ -306,3 +310,53 @@ Some of the common operations. ``` [/]> export templates/dashboards/dashboard ~/dashboard.json ``` + +## Basic Wildcard Support for Remote Resources + +Caveats/Notes +* The following is only supported when the `export` source or or `import` destination is a `remote` resource. +* The grafcli `ls` command is not able to navigate the `exports` hierarchy beyond the first level. +* All wildcards operate on the doc name/dashboard name (aka slug) not the actual filename, hence the .json part is not considered + +The `export` and `import` command have support for Bash style wildcards (e.g. ? and *) within the names of dashboards to export and import. The command be run from the CLI or from batch mode (take care when passing absolute paths with * in them from the shell, you may need to use "" around the path) + +The `export` command may be used to bulk export every dashboard or a selection of dashboards from some host xyz, and then place them either into a locally specified directory, an absolute directory path, or a newly created subdirectory of the `data-dir` named `exports/xyz`. Example usage follows: + +* Export all dashboards from host xyz, placing them into ./xyz_exports + +``` +[/]> export remote/xyz/* xyz_exports +``` + +* Export all dashboards matching the name `db*exporter*` from host xyz, placing them into `/home/someone/somewhere/exports` + +``` +[/]> export remote/xyz/*exporter* /home/someone/somewhere/exports +``` + +* Export all dashboards matching the name `thing?` from host xyz, (auto)placing them into `/exports/xyz` (this is a convenience shortcut) + +``` +[/]> export remote/xyz/thing? +``` + +The `import` command supports the reverse operation, where we may wish to import (upload) a selection of dashboards to some other host. In this case, it is essential that the configuration item `nullify_dbid` is set to `on` in order to prevent errors when importing. + +* Import all dashboards previously exported from host xyz, to host abc. + +``` +[/]> import exports/xyz/* remote/abc +``` + +* Import all dashboards matching the name `*hdfs*` previously exported from host xyz, to host abc. + +``` +[/]> import exports/xyz/*hdfs* remote/abc +``` + +* Import all dashboards matching the name `thing-?` from a direct ~ location, to host abc + +``` +[/]> import ~/dashboards/production/thing-? remote/abc +``` + diff --git a/grafcli.conf.example b/grafcli.conf.example index 8f1a46c..6258d75 100644 --- a/grafcli.conf.example +++ b/grafcli.conf.example @@ -4,6 +4,7 @@ history = ~/.grafcli_history verbose = off force = on colorize = on +nullify_dbid = off [resources] data-dir = ~/.grafcli diff --git a/grafcli/commands.py b/grafcli/commands.py index 791aef5..76d8da0 100644 --- a/grafcli/commands.py +++ b/grafcli/commands.py @@ -1,3 +1,4 @@ +import fnmatch import os import re import json @@ -7,7 +8,7 @@ from climb.config import config from climb.commands import Commands, command, completers from climb.exceptions import CLIException -from climb.paths import format_path, split_path, ROOT_PATH +from climb.paths import format_path, split_path, ROOT_PATH, SEPARATOR from grafcli.documents import Document, Dashboard, Row, Panel from grafcli.exceptions import CommandCancelled @@ -208,21 +209,115 @@ def restore(self, system_path, path): shutil.rmtree(tmp_dir) + def file_export_all(self, path_wildcard, system_path=None): + """ Perform a bulk export + - path_wildcard is typically going to be of the form + remote/abc/* or remote/abc/xyz*dash* + where Unix filename matching is performed against the doc_name + - system_path may be of the form of a local directory + name, an absolute path to a director, or, if ommitted + data-dir/exports/abc will be created and used. + """ + path_parts = split_path(path_wildcard) + if len(path_parts) != 3: + raise CLIException("Export all needs a host (wildcard hosts not yet supported)") + + location = path_parts[0] + host = path_parts[1] + wildcard = path_parts[2] + + if location != "remote": + raise CLIException("Export all is only available for remote resources") + + # Is this shortcut desirable? + if not system_path: + system_path = self._resources.local_system_path("exports", host) + + os.makedirs(system_path, exist_ok=True) + documents = self._resources.list(SEPARATOR.join(path_parts[0:2])) + for doc_name in documents: + if not fnmatch.fnmatch(doc_name, wildcard): + continue + file_path = os.path.join(system_path, to_file_format(doc_name)) + get_path = os.path.join(location, host, doc_name) + doc = self._resources.get(get_path) + with open(file_path, 'w') as file: + file.write(json_pretty(doc.source)) + self._cli.log("export: {} -> {}", get_path, file_path) + @command @completers('path', 'system_path') def file_export(self, path, system_path): + if not path: + raise CLIException("No path provided") + + parts = split_path(path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_export_all(path, system_path) + return + + if not system_path: + raise CLIException("No system path provided") + path = format_path(self._cli.current_path, path) system_path = os.path.expanduser(system_path) document = self._resources.get(path) - with open(system_path, 'w') as file: file.write(json_pretty(document.source)) self._cli.log("export: {} -> {}", path, system_path) + def file_import_all(self, system_path_wildcard, path): + """ Perform a bulk import + - system_path_wildcard may be of the form + /home/abc/exports/xyz/* or exports/xyz/*things* + where Unix filename matching is performed on the dashboard name + - path is typically going to be of the form + remote/abc + """ + + # don't use split_path for system_path_wildcard to preserve any leading / + system_parts = system_path_wildcard.split(SEPARATOR) + path_parts = split_path(path) + if len(path_parts) != 2: + raise CLIException("Import all needs a host (wildcard hosts not yet supported)") + + wildcard = system_parts[-1] + system_path = SEPARATOR.join(system_parts[0:-1]) + location = path_parts[0] + + if location != "remote": + raise CLIException("Import all is only available for remote resources") + + abs_system_path = os.path.expanduser(system_path) + + # check if it's an absolute path, if not check shortcut for 'exports' + if not os.path.isdir(abs_system_path): + abs_system_path = self._resources.local_system_path(path=system_path) + if not os.path.isdir(abs_system_path): + raise CLIException("Cannot find source path {}".format(system_path_wildcard)) + + dst_path = format_path(self._cli.current_path, path) + dashboards = self._resources.list_path(abs_system_path) + for dashboard_name in dashboards: + if not fnmatch.fnmatch(dashboard_name, wildcard): + continue + + file_path = os.path.join(abs_system_path, to_file_format(dashboard_name)) + with open(file_path, 'r') as file: + content = file.read() + document = Document.from_source(json.loads(content)) + self._resources.save(dst_path, document) + self._cli.log("import: {} -> {}/{}", file_path, dst_path, dashboard_name) + @command @completers('system_path', 'path') def file_import(self, system_path, path, match_slug=False): + parts = split_path(system_path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_import_all(system_path, path) + return + system_path = os.path.expanduser(system_path) path = format_path(self._cli.current_path, path) diff --git a/grafcli/resources/local.py b/grafcli/resources/local.py index a163407..3b6589d 100644 --- a/grafcli/resources/local.py +++ b/grafcli/resources/local.py @@ -4,4 +4,8 @@ class LocalResources(CommonResources): def __init__(self, local_dir): + self.local_dir = local_dir self._storage = SystemStorage(local_dir) + + def list_path(self, path): + return self._storage.list(path) diff --git a/grafcli/resources/resources.py b/grafcli/resources/resources.py index 210835a..5968d6b 100644 --- a/grafcli/resources/resources.py +++ b/grafcli/resources/resources.py @@ -1,3 +1,4 @@ +import os from climb.config import config from climb.paths import split_path @@ -6,14 +7,15 @@ from grafcli.resources.templates import DashboardsTemplates, RowsTemplates, PanelTemplates, CATEGORIES from grafcli.resources.local import LocalResources -LOCAL_DIR = 'backups' - +BACKUP_DIR = 'backups' +EXPORT_DIR = 'exports' class Resources(object): def __init__(self): self._resources = { - 'backups': LocalResources(LOCAL_DIR), + 'backups': LocalResources(BACKUP_DIR), + 'exports': LocalResources(EXPORT_DIR), 'remote': {}, 'templates': { 'dashboards': DashboardsTemplates(), @@ -22,6 +24,21 @@ def __init__(self): }, } + def local_system_path(self, resource=None, path=None): + """ Lookup the expanded local system path for some + resource and/or path + """ + p = os.path.join(config['resources']['data-dir']) + if resource: + if resource not in self._resources: + raise InvalidPath("Invalid resource {}".format(resource)) + p = os.path.join(config['resources']['data-dir'], + self._resources[resource].local_dir) + if path is not None: + p = os.path.join(p, path) + + return os.path.expanduser(p) + def list(self, path): """Returns list of sub-nodes for given path.""" try: @@ -37,6 +54,12 @@ def list(self, path): return manager.list(*parts) + def list_path(self, path): + """Use LocalResources to list files + in a SystemStorage path + """ + return LocalResources("").list_path(path) + def get(self, path): """Returns resource data.""" manager, parts = self._parse_path(path) diff --git a/grafcli/storage/api.py b/grafcli/storage/api.py index 981890d..aeef181 100644 --- a/grafcli/storage/api.py +++ b/grafcli/storage/api.py @@ -53,6 +53,12 @@ def save(self, dashboard_id, dashboard): raise data["dashboard"]["id"] = None + if config['grafcli'].getboolean("nullify_dbid"): + # No matter whether the db exists, nullify the ID to prevent + # 404 not found when moving a dashboard from some host xyz + # to host abc (else the below post will 404) + data["dashboard"]["id"] = None + self._call('POST', 'dashboards/db', data) def remove(self, dashboard_id): diff --git a/grafcli/storage/system.py b/grafcli/storage/system.py index 1efdb43..ff390be 100644 --- a/grafcli/storage/system.py +++ b/grafcli/storage/system.py @@ -15,8 +15,11 @@ def __init__(self, base_dir): self._base_dir = base_dir makepath(self._base_dir) - def list(self): - return list_files(self._base_dir) + def list(self, path=None): + list_path = self._base_dir + if path: + list_path = os.path.join(self._base_dir, path) + return list_files(list_path) def get(self, dashboard_id): try: diff --git a/tests/test_resources.py b/tests/test_resources.py index b41cf06..47a1e6e 100755 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -30,7 +30,7 @@ def tearDown(self): def test_list(self): r = Resources() - self.assertEqual(r.list(None), ['backups', 'remote', 'templates']) + self.assertEqual(r.list(None), ['backups', 'exports', 'remote', 'templates']) self.assertEqual(r.list('remote'), ['localhost']) self.assertEqual(r.list('templates'), ('dashboards', 'rows', 'panels'))