From 5aa649d7bfbcd23cefad10efe0075873efd54b01 Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 00:22:29 +0800 Subject: [PATCH 1/8] Add files via upload --- commands.py | 351 +++++++++++++++++++++++++++++++++++++++++++ grafcli.conf.example | 1 + 2 files changed, 352 insertions(+) create mode 100644 commands.py diff --git a/commands.py b/commands.py new file mode 100644 index 0000000..dcb45e6 --- /dev/null +++ b/commands.py @@ -0,0 +1,351 @@ +import fnmatch +import os +import re +import json +import shutil +import tarfile +import tempfile +from climb.config import config +from climb.commands import Commands, command, completers +from climb.exceptions import CLIException +from climb.paths import format_path, split_path, ROOT_PATH, SEPARATOR + +from grafcli.documents import Document, Dashboard, Row, Panel +from grafcli.exceptions import CommandCancelled +from grafcli.resources import Resources +from grafcli.storage.system import to_file_format, from_file_format +from grafcli.utils import json_pretty + + +class GrafCommands(Commands): + + def __init__(self, cli): + super().__init__(cli) + self._resources = Resources() + + @command + @completers('path') + def ls(self, path=None): + path = format_path(self._cli.current_path, path) + + result = self._resources.list(path) + + return "\n".join(sorted(result)) + + @command + @completers('path') + def cd(self, path=None): + path = format_path(self._cli.current_path, path, default=ROOT_PATH) + + # No exception means correct path + self._resources.list(path) + self._cli.set_current_path(path) + + @command + @completers('path') + def cat(self, path): + path = format_path(self._cli.current_path, path) + + document = self._resources.get(path) + return json_pretty(document.source, colorize=config['grafcli'].getboolean('colorize')) + + @command + @completers('path') + def cp(self, source, destination, match_slug=False): + if len(source) < 2: + raise CLIException("No destination provided") + + destination = source.pop(-1) + destination_path = format_path(self._cli.current_path, destination) + + for path in source: + source_path = format_path(self._cli.current_path, path) + + document = self._resources.get(source_path) + if match_slug: + destination_path = self._match_slug(document, destination_path) + + self._resources.save(destination_path, document) + + self._cli.log("cp: {} -> {}", source_path, destination_path) + + @command + @completers('path') + def mv(self, source, destination, match_slug=False): + if len(source) < 2: + raise CLIException("No destination provided") + + destination = source.pop(-1) + destination_path = format_path(self._cli.current_path, destination) + + for path in source: + source_path = format_path(self._cli.current_path, path) + document = self._resources.get(source_path) + + if match_slug: + destination_path = self._match_slug(document, destination_path) + + self._resources.save(destination_path, document) + self._resources.remove(source_path) + + self._cli.log("mv: {} -> {}", source_path, destination_path) + + @command + @completers('path') + def rm(self, path): + path = format_path(self._cli.current_path, path) + self._resources.remove(path) + + self._cli.log("rm: {}", path) + + @command + @completers('path') + def template(self, path): + path = format_path(self._cli.current_path, path) + document = self._resources.get(path) + + if isinstance(document, Dashboard): + template = 'dashboards' + elif isinstance(document, Row): + template = 'rows' + elif isinstance(document, Panel): + template = 'panels' + else: + raise CLIException("Unknown document type: {}".format( + document.__class__.__name__)) + + template_path = "/templates/{}".format(template) + self._resources.save(template_path, document) + + self._cli.log("template: {} -> {}", path, template_path) + + @command + @completers('path') + def editor(self, path): + path = format_path(self._cli.current_path, path) + document = self._resources.get(path) + + tmp_file = tempfile.mktemp(suffix=".json") + + with open(tmp_file, 'w') as file: + file.write(json_pretty(document.source)) + + cmd = "{} {}".format(config['grafcli']['editor'], tmp_file) + exit_status = os.system(cmd) + + if not exit_status: + self._cli.log("Updating: {}".format(path)) + self.file_import(tmp_file, path) + + os.unlink(tmp_file) + + @command + @completers('path') + def pos(self, path, position): + if not path: + raise CLIException("No path provided") + + if not position: + raise CLIException("No position provided") + + path = format_path(self._cli.current_path, path) + parts = split_path(path) + + parent_path = '/'.join(parts[:-1]) + child = parts[-1] + + parent = self._resources.get(parent_path) + parent.move_child(child, position) + + self._resources.save(parent_path, parent) + + @command + @completers('path', 'system_path') + def backup(self, path, system_path): + if not path: + raise CLIException("No path provided") + + if not system_path: + raise CLIException("No system path provided") + + path = format_path(self._cli.current_path, path) + system_path = os.path.expanduser(system_path) + + documents = self._resources.list(path) + if not documents: + raise CLIException("Nothing to backup") + + tmp_dir = tempfile.mkdtemp() + archive = tarfile.open(name=system_path, mode="w:gz") + + for doc_name in documents: + file_name = to_file_format(doc_name) + file_path = os.path.join(tmp_dir, file_name) + doc_path = os.path.join(path, doc_name) + + self.file_export(doc_path, file_path) + archive.add(file_path, arcname=file_name) + + archive.close() + shutil.rmtree(tmp_dir) + + @command + @completers('system_path', 'path') + def restore(self, system_path, path): + system_path = os.path.expanduser(system_path) + path = format_path(self._cli.current_path, path) + + tmp_dir = tempfile.mkdtemp() + with tarfile.open(name=system_path, mode="r:gz") as archive: + archive.extractall(path=tmp_dir) + + for name in os.listdir(tmp_dir): + try: + file_path = os.path.join(tmp_dir, name) + doc_path = os.path.join(path, from_file_format(name)) + self.file_import(file_path, doc_path) + except CommandCancelled: + pass + + shutil.rmtree(tmp_dir) + + def file_export_all(self, path_wildcard, system_path=None): + """ Perform a bulk export + - path_wildcard is typically going to be of the form + remote/abc/* or remote/abc/xyz*dash* + where Unix filename matching is performed on the wildcard + - system_path may be of the form of a local directory + name, an absolute path to a director, or, if ommitted + data-dir/exports/abc will be created and used. + """ + path_parts = split_path(path_wildcard) + if len(path_parts) != 3: + raise CLIException("Export all needs a host (wildcard hosts not yet supported)") + + location = path_parts[0] + host = path_parts[1] + wildcard = path_parts[2] + + if location != "remote": + raise CLIException("Export all is only available for remote resources") + + # Is this shortcut desirable? + if not system_path: + system_path = self._resources.local_system_path("exports", host) + + os.makedirs(system_path, exist_ok=True) + documents = self._resources.list(SEPARATOR.join(path_parts[0:2])) + for doc_name in documents: + file_name = to_file_format(doc_name) + if not fnmatch.fnmatch(file_name, wildcard): + continue + file_path = os.path.join(system_path, file_name) + get_path = os.path.join(location, host, doc_name) + doc = self._resources.get(get_path) + with open(file_path, 'w') as file: + file.write(json_pretty(doc.source)) + self._cli.log("export: {} -> {}", get_path, file_path) + + @command + @completers('path', 'system_path') + def file_export(self, path, system_path): + if not path: + raise CLIException("No path provided") + + parts = split_path(path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_export_all(path, system_path) + return + + if not system_path: + raise CLIException("No system path provided") + + path = format_path(self._cli.current_path, path) + system_path = os.path.expanduser(system_path) + document = self._resources.get(path) + with open(system_path, 'w') as file: + file.write(json_pretty(document.source)) + + self._cli.log("export: {} -> {}", path, system_path) + + def file_import_all(self, system_path_wildcard, path): + """ Perform a bulk import + - system_path_wildcard may be of the form + /home/abc/exports/xyz/* or exports/xyz/*things* + where Unix filename matching is performed on the wildcard + - path is typically going to be of the form + remote/abc + """ + + # don't use split_path for system_path_wildcard to preserve any leading / + system_parts = system_path_wildcard.split(SEPARATOR) + path_parts = split_path(path) + if len(path_parts) != 2: + raise CLIException("Import all needs a host (wildcard hosts not yet supported)") + + wildcard = system_parts[-1] + system_path = SEPARATOR.join(system_parts[0:-1]) + location = path_parts[0] + + if location != "remote": + raise CLIException("Import all is only available for remote resources") + + abs_system_path = os.path.expanduser(system_path) + + # check if it's an absolute path, if not check shortcut for 'exports' + if not os.path.isdir(abs_system_path): + abs_system_path = self._resources.local_system_path(path=system_path) + if not os.path.isdir(abs_system_path): + raise CLIException("Cannot find source path {}".format(system_path_wildcard)) + + dst_path = format_path(self._cli.current_path, path) + dashboards = self._resources.list_path(abs_system_path) + for dashboard_name in dashboards: + file_name = to_file_format(dashboard_name) + if not fnmatch.fnmatch(file_name, wildcard): + continue + file_path = os.path.join(abs_system_path, file_name) + + with open(file_path, 'r') as file: + content = file.read() + document = Document.from_source(json.loads(content)) + self._resources.save(dst_path, document) + self._cli.log("import: {} -> {}", file_path, dst_path) + + @command + @completers('system_path', 'path') + def file_import(self, system_path, path, match_slug=False): + parts = split_path(system_path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_import_all(system_path, path) + return + + system_path = os.path.expanduser(system_path) + path = format_path(self._cli.current_path, path) + + with open(system_path, 'r') as file: + content = file.read() + + document = Document.from_source(json.loads(content)) + + if match_slug: + path = self._match_slug(document, path) + + self._resources.save(path, document) + + self._cli.log("import: {} -> {}", system_path, path) + + def _match_slug(self, document, destination): + pattern = re.compile(r'^\d+-{}$'.format(document.slug)) + + children = self._resources.list(destination) + matches = [child for child in children + if pattern.search(child)] + + if not matches: + return destination + + if len(matches) > 2: + raise CLIException("Too many matching slugs, be more specific") + + return "{}/{}".format(destination, matches[0]) diff --git a/grafcli.conf.example b/grafcli.conf.example index 8f1a46c..6258d75 100644 --- a/grafcli.conf.example +++ b/grafcli.conf.example @@ -4,6 +4,7 @@ history = ~/.grafcli_history verbose = off force = on colorize = on +nullify_dbid = off [resources] data-dir = ~/.grafcli From e4418414b326a07382401136c6035c9df13b27f2 Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 00:23:39 +0800 Subject: [PATCH 2/8] Delete commands.py --- commands.py | 351 ---------------------------------------------------- 1 file changed, 351 deletions(-) delete mode 100644 commands.py diff --git a/commands.py b/commands.py deleted file mode 100644 index dcb45e6..0000000 --- a/commands.py +++ /dev/null @@ -1,351 +0,0 @@ -import fnmatch -import os -import re -import json -import shutil -import tarfile -import tempfile -from climb.config import config -from climb.commands import Commands, command, completers -from climb.exceptions import CLIException -from climb.paths import format_path, split_path, ROOT_PATH, SEPARATOR - -from grafcli.documents import Document, Dashboard, Row, Panel -from grafcli.exceptions import CommandCancelled -from grafcli.resources import Resources -from grafcli.storage.system import to_file_format, from_file_format -from grafcli.utils import json_pretty - - -class GrafCommands(Commands): - - def __init__(self, cli): - super().__init__(cli) - self._resources = Resources() - - @command - @completers('path') - def ls(self, path=None): - path = format_path(self._cli.current_path, path) - - result = self._resources.list(path) - - return "\n".join(sorted(result)) - - @command - @completers('path') - def cd(self, path=None): - path = format_path(self._cli.current_path, path, default=ROOT_PATH) - - # No exception means correct path - self._resources.list(path) - self._cli.set_current_path(path) - - @command - @completers('path') - def cat(self, path): - path = format_path(self._cli.current_path, path) - - document = self._resources.get(path) - return json_pretty(document.source, colorize=config['grafcli'].getboolean('colorize')) - - @command - @completers('path') - def cp(self, source, destination, match_slug=False): - if len(source) < 2: - raise CLIException("No destination provided") - - destination = source.pop(-1) - destination_path = format_path(self._cli.current_path, destination) - - for path in source: - source_path = format_path(self._cli.current_path, path) - - document = self._resources.get(source_path) - if match_slug: - destination_path = self._match_slug(document, destination_path) - - self._resources.save(destination_path, document) - - self._cli.log("cp: {} -> {}", source_path, destination_path) - - @command - @completers('path') - def mv(self, source, destination, match_slug=False): - if len(source) < 2: - raise CLIException("No destination provided") - - destination = source.pop(-1) - destination_path = format_path(self._cli.current_path, destination) - - for path in source: - source_path = format_path(self._cli.current_path, path) - document = self._resources.get(source_path) - - if match_slug: - destination_path = self._match_slug(document, destination_path) - - self._resources.save(destination_path, document) - self._resources.remove(source_path) - - self._cli.log("mv: {} -> {}", source_path, destination_path) - - @command - @completers('path') - def rm(self, path): - path = format_path(self._cli.current_path, path) - self._resources.remove(path) - - self._cli.log("rm: {}", path) - - @command - @completers('path') - def template(self, path): - path = format_path(self._cli.current_path, path) - document = self._resources.get(path) - - if isinstance(document, Dashboard): - template = 'dashboards' - elif isinstance(document, Row): - template = 'rows' - elif isinstance(document, Panel): - template = 'panels' - else: - raise CLIException("Unknown document type: {}".format( - document.__class__.__name__)) - - template_path = "/templates/{}".format(template) - self._resources.save(template_path, document) - - self._cli.log("template: {} -> {}", path, template_path) - - @command - @completers('path') - def editor(self, path): - path = format_path(self._cli.current_path, path) - document = self._resources.get(path) - - tmp_file = tempfile.mktemp(suffix=".json") - - with open(tmp_file, 'w') as file: - file.write(json_pretty(document.source)) - - cmd = "{} {}".format(config['grafcli']['editor'], tmp_file) - exit_status = os.system(cmd) - - if not exit_status: - self._cli.log("Updating: {}".format(path)) - self.file_import(tmp_file, path) - - os.unlink(tmp_file) - - @command - @completers('path') - def pos(self, path, position): - if not path: - raise CLIException("No path provided") - - if not position: - raise CLIException("No position provided") - - path = format_path(self._cli.current_path, path) - parts = split_path(path) - - parent_path = '/'.join(parts[:-1]) - child = parts[-1] - - parent = self._resources.get(parent_path) - parent.move_child(child, position) - - self._resources.save(parent_path, parent) - - @command - @completers('path', 'system_path') - def backup(self, path, system_path): - if not path: - raise CLIException("No path provided") - - if not system_path: - raise CLIException("No system path provided") - - path = format_path(self._cli.current_path, path) - system_path = os.path.expanduser(system_path) - - documents = self._resources.list(path) - if not documents: - raise CLIException("Nothing to backup") - - tmp_dir = tempfile.mkdtemp() - archive = tarfile.open(name=system_path, mode="w:gz") - - for doc_name in documents: - file_name = to_file_format(doc_name) - file_path = os.path.join(tmp_dir, file_name) - doc_path = os.path.join(path, doc_name) - - self.file_export(doc_path, file_path) - archive.add(file_path, arcname=file_name) - - archive.close() - shutil.rmtree(tmp_dir) - - @command - @completers('system_path', 'path') - def restore(self, system_path, path): - system_path = os.path.expanduser(system_path) - path = format_path(self._cli.current_path, path) - - tmp_dir = tempfile.mkdtemp() - with tarfile.open(name=system_path, mode="r:gz") as archive: - archive.extractall(path=tmp_dir) - - for name in os.listdir(tmp_dir): - try: - file_path = os.path.join(tmp_dir, name) - doc_path = os.path.join(path, from_file_format(name)) - self.file_import(file_path, doc_path) - except CommandCancelled: - pass - - shutil.rmtree(tmp_dir) - - def file_export_all(self, path_wildcard, system_path=None): - """ Perform a bulk export - - path_wildcard is typically going to be of the form - remote/abc/* or remote/abc/xyz*dash* - where Unix filename matching is performed on the wildcard - - system_path may be of the form of a local directory - name, an absolute path to a director, or, if ommitted - data-dir/exports/abc will be created and used. - """ - path_parts = split_path(path_wildcard) - if len(path_parts) != 3: - raise CLIException("Export all needs a host (wildcard hosts not yet supported)") - - location = path_parts[0] - host = path_parts[1] - wildcard = path_parts[2] - - if location != "remote": - raise CLIException("Export all is only available for remote resources") - - # Is this shortcut desirable? - if not system_path: - system_path = self._resources.local_system_path("exports", host) - - os.makedirs(system_path, exist_ok=True) - documents = self._resources.list(SEPARATOR.join(path_parts[0:2])) - for doc_name in documents: - file_name = to_file_format(doc_name) - if not fnmatch.fnmatch(file_name, wildcard): - continue - file_path = os.path.join(system_path, file_name) - get_path = os.path.join(location, host, doc_name) - doc = self._resources.get(get_path) - with open(file_path, 'w') as file: - file.write(json_pretty(doc.source)) - self._cli.log("export: {} -> {}", get_path, file_path) - - @command - @completers('path', 'system_path') - def file_export(self, path, system_path): - if not path: - raise CLIException("No path provided") - - parts = split_path(path) - if "*" in parts[-1] or "?" in parts[-1]: - self.file_export_all(path, system_path) - return - - if not system_path: - raise CLIException("No system path provided") - - path = format_path(self._cli.current_path, path) - system_path = os.path.expanduser(system_path) - document = self._resources.get(path) - with open(system_path, 'w') as file: - file.write(json_pretty(document.source)) - - self._cli.log("export: {} -> {}", path, system_path) - - def file_import_all(self, system_path_wildcard, path): - """ Perform a bulk import - - system_path_wildcard may be of the form - /home/abc/exports/xyz/* or exports/xyz/*things* - where Unix filename matching is performed on the wildcard - - path is typically going to be of the form - remote/abc - """ - - # don't use split_path for system_path_wildcard to preserve any leading / - system_parts = system_path_wildcard.split(SEPARATOR) - path_parts = split_path(path) - if len(path_parts) != 2: - raise CLIException("Import all needs a host (wildcard hosts not yet supported)") - - wildcard = system_parts[-1] - system_path = SEPARATOR.join(system_parts[0:-1]) - location = path_parts[0] - - if location != "remote": - raise CLIException("Import all is only available for remote resources") - - abs_system_path = os.path.expanduser(system_path) - - # check if it's an absolute path, if not check shortcut for 'exports' - if not os.path.isdir(abs_system_path): - abs_system_path = self._resources.local_system_path(path=system_path) - if not os.path.isdir(abs_system_path): - raise CLIException("Cannot find source path {}".format(system_path_wildcard)) - - dst_path = format_path(self._cli.current_path, path) - dashboards = self._resources.list_path(abs_system_path) - for dashboard_name in dashboards: - file_name = to_file_format(dashboard_name) - if not fnmatch.fnmatch(file_name, wildcard): - continue - file_path = os.path.join(abs_system_path, file_name) - - with open(file_path, 'r') as file: - content = file.read() - document = Document.from_source(json.loads(content)) - self._resources.save(dst_path, document) - self._cli.log("import: {} -> {}", file_path, dst_path) - - @command - @completers('system_path', 'path') - def file_import(self, system_path, path, match_slug=False): - parts = split_path(system_path) - if "*" in parts[-1] or "?" in parts[-1]: - self.file_import_all(system_path, path) - return - - system_path = os.path.expanduser(system_path) - path = format_path(self._cli.current_path, path) - - with open(system_path, 'r') as file: - content = file.read() - - document = Document.from_source(json.loads(content)) - - if match_slug: - path = self._match_slug(document, path) - - self._resources.save(path, document) - - self._cli.log("import: {} -> {}", system_path, path) - - def _match_slug(self, document, destination): - pattern = re.compile(r'^\d+-{}$'.format(document.slug)) - - children = self._resources.list(destination) - matches = [child for child in children - if pattern.search(child)] - - if not matches: - return destination - - if len(matches) > 2: - raise CLIException("Too many matching slugs, be more specific") - - return "{}/{}".format(destination, matches[0]) From 75e9bdd3306d9625ac00439438220fb6174ead80 Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 00:29:45 +0800 Subject: [PATCH 3/8] Bash wildcard import and export currently only for remote resources. --- grafcli/commands.py | 101 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/grafcli/commands.py b/grafcli/commands.py index 791aef5..dcb45e6 100644 --- a/grafcli/commands.py +++ b/grafcli/commands.py @@ -1,3 +1,4 @@ +import fnmatch import os import re import json @@ -7,7 +8,7 @@ from climb.config import config from climb.commands import Commands, command, completers from climb.exceptions import CLIException -from climb.paths import format_path, split_path, ROOT_PATH +from climb.paths import format_path, split_path, ROOT_PATH, SEPARATOR from grafcli.documents import Document, Dashboard, Row, Panel from grafcli.exceptions import CommandCancelled @@ -208,21 +209,117 @@ def restore(self, system_path, path): shutil.rmtree(tmp_dir) + def file_export_all(self, path_wildcard, system_path=None): + """ Perform a bulk export + - path_wildcard is typically going to be of the form + remote/abc/* or remote/abc/xyz*dash* + where Unix filename matching is performed on the wildcard + - system_path may be of the form of a local directory + name, an absolute path to a director, or, if ommitted + data-dir/exports/abc will be created and used. + """ + path_parts = split_path(path_wildcard) + if len(path_parts) != 3: + raise CLIException("Export all needs a host (wildcard hosts not yet supported)") + + location = path_parts[0] + host = path_parts[1] + wildcard = path_parts[2] + + if location != "remote": + raise CLIException("Export all is only available for remote resources") + + # Is this shortcut desirable? + if not system_path: + system_path = self._resources.local_system_path("exports", host) + + os.makedirs(system_path, exist_ok=True) + documents = self._resources.list(SEPARATOR.join(path_parts[0:2])) + for doc_name in documents: + file_name = to_file_format(doc_name) + if not fnmatch.fnmatch(file_name, wildcard): + continue + file_path = os.path.join(system_path, file_name) + get_path = os.path.join(location, host, doc_name) + doc = self._resources.get(get_path) + with open(file_path, 'w') as file: + file.write(json_pretty(doc.source)) + self._cli.log("export: {} -> {}", get_path, file_path) + @command @completers('path', 'system_path') def file_export(self, path, system_path): + if not path: + raise CLIException("No path provided") + + parts = split_path(path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_export_all(path, system_path) + return + + if not system_path: + raise CLIException("No system path provided") + path = format_path(self._cli.current_path, path) system_path = os.path.expanduser(system_path) document = self._resources.get(path) - with open(system_path, 'w') as file: file.write(json_pretty(document.source)) self._cli.log("export: {} -> {}", path, system_path) + def file_import_all(self, system_path_wildcard, path): + """ Perform a bulk import + - system_path_wildcard may be of the form + /home/abc/exports/xyz/* or exports/xyz/*things* + where Unix filename matching is performed on the wildcard + - path is typically going to be of the form + remote/abc + """ + + # don't use split_path for system_path_wildcard to preserve any leading / + system_parts = system_path_wildcard.split(SEPARATOR) + path_parts = split_path(path) + if len(path_parts) != 2: + raise CLIException("Import all needs a host (wildcard hosts not yet supported)") + + wildcard = system_parts[-1] + system_path = SEPARATOR.join(system_parts[0:-1]) + location = path_parts[0] + + if location != "remote": + raise CLIException("Import all is only available for remote resources") + + abs_system_path = os.path.expanduser(system_path) + + # check if it's an absolute path, if not check shortcut for 'exports' + if not os.path.isdir(abs_system_path): + abs_system_path = self._resources.local_system_path(path=system_path) + if not os.path.isdir(abs_system_path): + raise CLIException("Cannot find source path {}".format(system_path_wildcard)) + + dst_path = format_path(self._cli.current_path, path) + dashboards = self._resources.list_path(abs_system_path) + for dashboard_name in dashboards: + file_name = to_file_format(dashboard_name) + if not fnmatch.fnmatch(file_name, wildcard): + continue + file_path = os.path.join(abs_system_path, file_name) + + with open(file_path, 'r') as file: + content = file.read() + document = Document.from_source(json.loads(content)) + self._resources.save(dst_path, document) + self._cli.log("import: {} -> {}", file_path, dst_path) + @command @completers('system_path', 'path') def file_import(self, system_path, path, match_slug=False): + parts = split_path(system_path) + if "*" in parts[-1] or "?" in parts[-1]: + self.file_import_all(system_path, path) + return + system_path = os.path.expanduser(system_path) path = format_path(self._cli.current_path, path) From 1d84d1c335bcf99361c31e3a6a9454ef0509130c Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 00:30:45 +0800 Subject: [PATCH 4/8] Bash wildcard import and export --- grafcli/resources/local.py | 4 ++++ grafcli/resources/resources.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/grafcli/resources/local.py b/grafcli/resources/local.py index a163407..3b6589d 100644 --- a/grafcli/resources/local.py +++ b/grafcli/resources/local.py @@ -4,4 +4,8 @@ class LocalResources(CommonResources): def __init__(self, local_dir): + self.local_dir = local_dir self._storage = SystemStorage(local_dir) + + def list_path(self, path): + return self._storage.list(path) diff --git a/grafcli/resources/resources.py b/grafcli/resources/resources.py index 210835a..5968d6b 100644 --- a/grafcli/resources/resources.py +++ b/grafcli/resources/resources.py @@ -1,3 +1,4 @@ +import os from climb.config import config from climb.paths import split_path @@ -6,14 +7,15 @@ from grafcli.resources.templates import DashboardsTemplates, RowsTemplates, PanelTemplates, CATEGORIES from grafcli.resources.local import LocalResources -LOCAL_DIR = 'backups' - +BACKUP_DIR = 'backups' +EXPORT_DIR = 'exports' class Resources(object): def __init__(self): self._resources = { - 'backups': LocalResources(LOCAL_DIR), + 'backups': LocalResources(BACKUP_DIR), + 'exports': LocalResources(EXPORT_DIR), 'remote': {}, 'templates': { 'dashboards': DashboardsTemplates(), @@ -22,6 +24,21 @@ def __init__(self): }, } + def local_system_path(self, resource=None, path=None): + """ Lookup the expanded local system path for some + resource and/or path + """ + p = os.path.join(config['resources']['data-dir']) + if resource: + if resource not in self._resources: + raise InvalidPath("Invalid resource {}".format(resource)) + p = os.path.join(config['resources']['data-dir'], + self._resources[resource].local_dir) + if path is not None: + p = os.path.join(p, path) + + return os.path.expanduser(p) + def list(self, path): """Returns list of sub-nodes for given path.""" try: @@ -37,6 +54,12 @@ def list(self, path): return manager.list(*parts) + def list_path(self, path): + """Use LocalResources to list files + in a SystemStorage path + """ + return LocalResources("").list_path(path) + def get(self, path): """Returns resource data.""" manager, parts = self._parse_path(path) From c72657b88abdcc5abfc3635d7f12edcda466ffa3 Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 00:31:44 +0800 Subject: [PATCH 5/8] Bash wildcard import and export --- grafcli/storage/api.py | 6 ++++++ grafcli/storage/system.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/grafcli/storage/api.py b/grafcli/storage/api.py index 981890d..aeef181 100644 --- a/grafcli/storage/api.py +++ b/grafcli/storage/api.py @@ -53,6 +53,12 @@ def save(self, dashboard_id, dashboard): raise data["dashboard"]["id"] = None + if config['grafcli'].getboolean("nullify_dbid"): + # No matter whether the db exists, nullify the ID to prevent + # 404 not found when moving a dashboard from some host xyz + # to host abc (else the below post will 404) + data["dashboard"]["id"] = None + self._call('POST', 'dashboards/db', data) def remove(self, dashboard_id): diff --git a/grafcli/storage/system.py b/grafcli/storage/system.py index 1efdb43..ff390be 100644 --- a/grafcli/storage/system.py +++ b/grafcli/storage/system.py @@ -15,8 +15,11 @@ def __init__(self, base_dir): self._base_dir = base_dir makepath(self._base_dir) - def list(self): - return list_files(self._base_dir) + def list(self, path=None): + list_path = self._base_dir + if path: + list_path = os.path.join(self._base_dir, path) + return list_files(list_path) def get(self, dashboard_id): try: From 129b749de9e137567defba8bb25b8eff7665dd5a Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 01:18:54 +0800 Subject: [PATCH 6/8] Update readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 12f6941..8f7b31c 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ history = ~/.grafcli_history verbose = off # Answer 'yes' to all overwrite prompts. force = on +# Forcibly nullify the dashboard ID when importing a dashboard +# that was exported from some other host. Enable this if you +# wish to move dashboards between hosts using export/import. +nullify_dbid = off [resources] # Directory where all local data will be stored (including backups). @@ -306,3 +310,53 @@ Some of the common operations. ``` [/]> export templates/dashboards/dashboard ~/dashboard.json ``` + +## Basic Wildcard Support for Remote Resources + +Caveats/Notes +* The following is only supported when the `export` source or or `import` destination is a `remote` resource. +* The grafcli `ls` command is not able to navigate the `exports` hierarchy beyond the first level. +* All wildcards operate on the doc name/dashboard name (aka slug) not the actual filename, hence the .json part is not considered + +The `export` and `import` command have support for Bash style wildcards (e.g. ? and *) within the names of dashboards to export and import. The command be run from the CLI or from batch mode (take care when passing absolute paths with * in them from the shell, you may need to use "" around the path) + +The `export` command may be used to bulk export every dashboard or a selection of dashboards from some host xyz, and then place them either into a locally specified directory, an absolute directory path, or a newly created subdirectory of the `data-dir` named `exports/xyz`. Example usage follows: + +* Export all dashboards from host xyz, placing them into ./xyz_exports + +``` +[/]> export remote/xyz/* xyz_exports +``` + +* Export all dashboards matching the name `db*exporter*` from host xyz, placing them into `/home/someone/somewhere/exports` + +``` +[/]> export remote/xyz/*exporter* /home/someone/somewhere/exports +``` + +* Export all dashboards matching the name `thing?` from host xyz, (auto)placing them into `/exports/xyz` (this is a convenience shortcut) + +``` +[/]> export remote/xyz/thing? +``` + +The `import` command supports the reverse operation, where we may wish to import (upload) a selection of dashboards to some other host. In this case, it is essential that the configuration item `nullify_dbid` is set to `on` in order to prevent errors when importing. + +* Import all dashboards previously exported from host xyz, to host abc. + +``` +[/]> import exports/xyz/* remote/abc +``` + +* Import all dashboards matching the name `*hdfs*` previously exported from host xyz, to host abc. + +``` +[/]> import exports/xyz/*hdfs* remote/abc +``` + +* Import all dashboards matching the name `thing-?` from a direct ~ location, to host abc + +``` +[/]> import ~/dashboards/production/thing-? remote/abc +``` + From 80f4cb8237bfe0a22216eb7cd048265dda0b753f Mon Sep 17 00:00:00 2001 From: jgoff Date: Fri, 5 May 2017 01:20:35 +0800 Subject: [PATCH 7/8] Bash wildcard import and export tweaks --- grafcli/commands.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/grafcli/commands.py b/grafcli/commands.py index dcb45e6..76d8da0 100644 --- a/grafcli/commands.py +++ b/grafcli/commands.py @@ -213,7 +213,7 @@ def file_export_all(self, path_wildcard, system_path=None): """ Perform a bulk export - path_wildcard is typically going to be of the form remote/abc/* or remote/abc/xyz*dash* - where Unix filename matching is performed on the wildcard + where Unix filename matching is performed against the doc_name - system_path may be of the form of a local directory name, an absolute path to a director, or, if ommitted data-dir/exports/abc will be created and used. @@ -235,11 +235,10 @@ def file_export_all(self, path_wildcard, system_path=None): os.makedirs(system_path, exist_ok=True) documents = self._resources.list(SEPARATOR.join(path_parts[0:2])) - for doc_name in documents: - file_name = to_file_format(doc_name) - if not fnmatch.fnmatch(file_name, wildcard): + for doc_name in documents: + if not fnmatch.fnmatch(doc_name, wildcard): continue - file_path = os.path.join(system_path, file_name) + file_path = os.path.join(system_path, to_file_format(doc_name)) get_path = os.path.join(location, host, doc_name) doc = self._resources.get(get_path) with open(file_path, 'w') as file: @@ -272,7 +271,7 @@ def file_import_all(self, system_path_wildcard, path): """ Perform a bulk import - system_path_wildcard may be of the form /home/abc/exports/xyz/* or exports/xyz/*things* - where Unix filename matching is performed on the wildcard + where Unix filename matching is performed on the dashboard name - path is typically going to be of the form remote/abc """ @@ -301,16 +300,15 @@ def file_import_all(self, system_path_wildcard, path): dst_path = format_path(self._cli.current_path, path) dashboards = self._resources.list_path(abs_system_path) for dashboard_name in dashboards: - file_name = to_file_format(dashboard_name) - if not fnmatch.fnmatch(file_name, wildcard): + if not fnmatch.fnmatch(dashboard_name, wildcard): continue - file_path = os.path.join(abs_system_path, file_name) + file_path = os.path.join(abs_system_path, to_file_format(dashboard_name)) with open(file_path, 'r') as file: content = file.read() document = Document.from_source(json.loads(content)) self._resources.save(dst_path, document) - self._cli.log("import: {} -> {}", file_path, dst_path) + self._cli.log("import: {} -> {}/{}", file_path, dst_path, dashboard_name) @command @completers('system_path', 'path') From 78588fb6d78df0713a81c249f9b591f198dae348 Mon Sep 17 00:00:00 2001 From: Jeff Date: Fri, 5 May 2017 14:26:17 +0800 Subject: [PATCH 8/8] Add exports to test_list --- tests/test_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resources.py b/tests/test_resources.py index b41cf06..47a1e6e 100755 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -30,7 +30,7 @@ def tearDown(self): def test_list(self): r = Resources() - self.assertEqual(r.list(None), ['backups', 'remote', 'templates']) + self.assertEqual(r.list(None), ['backups', 'exports', 'remote', 'templates']) self.assertEqual(r.list('remote'), ['localhost']) self.assertEqual(r.list('templates'), ('dashboards', 'rows', 'panels'))