From 5eb68686c9e888980c1312b6af3af1aac8626c9c Mon Sep 17 00:00:00 2001 From: necusjz Date: Fri, 6 Dec 2024 14:53:22 +0800 Subject: [PATCH 01/18] perf: no need to verify tree.json --- src/aaz_dev/command/api/_cmds.py | 37 ++++++-------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index d4d9e100..1e902317 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -167,7 +167,7 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): help="Path of `aaz` repository." ) def verify(): - def verify_command(file_path, node): + def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() @@ -175,32 +175,18 @@ def verify_command(file_path, node): for path in paths: json_path = os.path.join(Config.AAZ_PATH, os.path.splitext(path)[0][1:] + ".json") json_path = os.path.normpath(json_path) + if not os.path.exists(json_path): raise Exception(f"{json_path} defined in {file_path} is missing.") - with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: - model = json.load(fp) - group, command = " ".join(node.names[:-1]), node.names[-1] - for g in model["commandGroups"]: - if g["name"] == group: - if not any(cmd["name"] == command for cmd in g["commands"]): - raise Exception(f"There is no {command} command info in {json_path}.") - - break - model_set.add(json_path) - tmpl = get_templates()["command"] - if not tmpl.render(command=node) == content: - raise Exception(f"{file_path} cannot be rendered correctly.") - model_set = set() aaz = AAZSpecsManager() - stack = [(aaz.commands_folder, aaz.tree.root)] # root nodes + stack = [aaz.commands_folder] while stack: - curr_path, curr_node = stack.pop() - logger.info(f"Checking {curr_path}") + curr_path = stack.pop() if os.path.isdir(curr_path): readme_path = os.path.join(curr_path, "readme.md") if not os.path.exists(readme_path): @@ -227,13 +213,9 @@ def verify_command(file_path, node): diff = cmd_set - items or items - cmd_set raise Exception(f"Command info {diff} doesn't match in {readme_path}.") - groups = set(curr_node.commands.keys()) - if groups != items: - diff = groups - items or items - groups - raise Exception(f"Command info {diff} in tree.json doesn't match in {readme_path}.") - for file in files: - verify_command(os.path.join(curr_path, file), curr_node.commands[file[1:-3]]) + verify_command(os.path.join(curr_path, file)) + else: if len(items) != len(set(items)): raise Exception(f"{readme_path} has duplicate command group names.") @@ -245,13 +227,8 @@ def verify_command(file_path, node): diff = folders - items or items - folders raise Exception(f"Command group info {diff} doesn't match in {readme_path}.") - groups = set(curr_node.command_groups.keys()) - if groups != set(items): - diff = groups - items or items - groups - raise Exception(f"Command group info {diff} in tree.json doesn't match in {readme_path}.") - for folder in folders: - stack.append((os.path.join(curr_path, folder), curr_node.command_groups[folder])) + stack.append(os.path.join(curr_path, folder)) for root, dirs, files in os.walk(aaz.resources_folder): for file in files: From 942cd7d9e16868653dd1d461cc1d7c3145893ebc Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 16 Dec 2024 16:28:56 +0800 Subject: [PATCH 02/18] feat: verify json to commands --- src/aaz_dev/command/api/_cmds.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 1e902317..0da48997 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -171,14 +171,37 @@ def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() + folder = os.path.dirname(file_path) + curr_cmd = os.path.splitext(os.path.basename(file_path))[0][1:] + curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) + + # _command_name.md -> command_name + cmd_set = set(map(lambda x: x[1:-3], [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))])) + paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: json_path = os.path.join(Config.AAZ_PATH, os.path.splitext(path)[0][1:] + ".json") json_path = os.path.normpath(json_path) + if json_path in model_set: + continue + if not os.path.exists(json_path): raise Exception(f"{json_path} defined in {file_path} is missing.") + with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: + model = json.load(fp) + + for grp in model["commandGroups"]: + if grp["name"] == curr_grp: + if not any(cmd["name"] == curr_cmd for cmd in grp["commands"]): + raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + + if any(cmd["name"] not in cmd_set for cmd in grp["commands"]): + raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") + + break + model_set.add(json_path) model_set = set() From c927001c5547574a9b88d77a94482e6d68988a57 Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 17 Dec 2024 16:26:42 +0800 Subject: [PATCH 03/18] fix: support nested command groups --- src/aaz_dev/command/api/_cmds.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 0da48997..2f59ed26 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -192,15 +192,24 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) - for grp in model["commandGroups"]: - if grp["name"] == curr_grp: - if not any(cmd["name"] == curr_cmd for cmd in grp["commands"]): - raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + group = [] + while True: + try: + group.append(model["commandGroups"][0]["name"]) + if " ".join(group) == curr_grp: + break - if any(cmd["name"] not in cmd_set for cmd in grp["commands"]): - raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") + model = model["commandGroups"][0] - break + except KeyError: + raise Exception(f"{curr_grp} {curr_cmd} is redundant.") + + commands = model["commandGroups"][0]["commands"] + if not any(cmd["name"] == curr_cmd for cmd in commands): + raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + + if any(cmd["name"] not in cmd_set for cmd in commands): + raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") model_set.add(json_path) From c5b222f052b47933072449d4d477de3dd5ac2b72 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 18 Dec 2024 14:10:05 +0800 Subject: [PATCH 04/18] fix: group may contain many subgroups --- src/aaz_dev/command/api/_cmds.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 2f59ed26..104bd3dd 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -176,7 +176,7 @@ def verify_command(file_path): curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) # _command_name.md -> command_name - cmd_set = set(map(lambda x: x[1:-3], [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))])) + cmd_set = set(map(lambda x: x[1:-3], [i for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i))])) paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: @@ -192,19 +192,20 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) - group = [] - while True: + target = curr_grp + while target: try: - group.append(model["commandGroups"][0]["name"]) - if " ".join(group) == curr_grp: - break + for grp in model["commandGroups"]: + if target.startswith(grp["name"]): + target = target[len(grp["name"]):].strip() + model = grp - model = model["commandGroups"][0] + break except KeyError: raise Exception(f"{curr_grp} {curr_cmd} is redundant.") - commands = model["commandGroups"][0]["commands"] + commands = model["commands"] if not any(cmd["name"] == curr_cmd for cmd in commands): raise Exception(f"There is no {curr_cmd} command info in {json_path}.") From 8de427fd35ce6880bbaab66c86aa9ce67583fb72 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 19 Dec 2024 22:16:21 +0800 Subject: [PATCH 05/18] feat: add verify_resource --- src/aaz_dev/command/api/_cmds.py | 36 +++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 104bd3dd..c3c239f4 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -167,16 +167,29 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): help="Path of `aaz` repository." ) def verify(): + def verify_resource(model, path): + if "commandGroups" not in model: + return + + for grp in model["commandGroups"]: + base_path = os.path.join(path, *grp["name"].split()) + if not os.path.exists(base_path): + raise FileNotFoundError(base_path) + + for cmd in grp.get("commands", []): + file_path = os.path.join(base_path, f"_{cmd['name']}.md") + if not os.path.isfile(file_path): + raise FileNotFoundError(file_path) + + verify_resource(grp, base_path) + def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() - folder = os.path.dirname(file_path) + base_path = os.path.dirname(file_path) + curr_grp = " ".join(os.path.relpath(base_path, aaz.commands_folder).split(os.sep)) curr_cmd = os.path.splitext(os.path.basename(file_path))[0][1:] - curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) - - # _command_name.md -> command_name - cmd_set = set(map(lambda x: x[1:-3], [i for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i))])) paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: @@ -192,6 +205,12 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) + try: + verify_resource(model, aaz.commands_folder) + + except FileNotFoundError as e: + raise Exception(f"Cannot find {e} defined in {json_path}.") + target = curr_grp while target: try: @@ -203,15 +222,12 @@ def verify_command(file_path): break except KeyError: - raise Exception(f"{curr_grp} {curr_cmd} is redundant.") + raise Exception(f"{curr_grp} has no corresponding definition in {json_path}.") commands = model["commands"] if not any(cmd["name"] == curr_cmd for cmd in commands): raise Exception(f"There is no {curr_cmd} command info in {json_path}.") - if any(cmd["name"] not in cmd_set for cmd in commands): - raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") - model_set.add(json_path) model_set = set() @@ -263,7 +279,7 @@ def verify_command(file_path): for folder in folders: stack.append(os.path.join(curr_path, folder)) - for root, dirs, files in os.walk(aaz.resources_folder): + for root, _, files in os.walk(aaz.resources_folder): for file in files: if not file.endswith(".json") or file.startswith("client"): # support data-plane continue From fa549b7153f06bada519d753a195dfadee4c3968 Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 11:01:26 +0800 Subject: [PATCH 06/18] swagger rp support multiple readme files --- .../swagger/model/specs/_resource_provider.py | 85 ++++++++++--------- .../swagger/model/specs/_swagger_module.py | 52 ++++-------- .../swagger/model/specs/_swagger_specs.py | 10 +-- src/aaz_dev/swagger/utils/tools.py | 13 +++ src/aaz_dev/utils/readme_helper.py | 50 +++++++++++ 5 files changed, 131 insertions(+), 79 deletions(-) create mode 100644 src/aaz_dev/utils/readme_helper.py diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index 20e85578..a2b91dfd 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -7,22 +7,22 @@ import yaml -from swagger.utils.tools import swagger_resource_path_to_resource_id +from swagger.utils.tools import swagger_resource_path_to_resource_id, resolve_path_to_uri from ._resource import Resource, ResourceVersion from ._utils import map_path_2_repo - +from utils.readme_helper import parse_readme_file logger = logging.getLogger('backend') class OpenAPIResourceProvider: - def __init__(self, name, folder_path, readme_path, swagger_module): + def __init__(self, name, folder_path, readme_paths, swagger_module): self.name = name self.folder_path = folder_path - self._readme_path = readme_path + self._readme_paths = readme_paths self.swagger_module = swagger_module - if readme_path is None: + if not readme_paths: logger.warning(f"MissReadmeFile: {self} : {map_path_2_repo(folder_path)}") self._tags = None self._resource_map = None @@ -74,6 +74,12 @@ def get_resource_map_by_tag(self, tag): resource_map[resource.id][resource.version] = resource return resource_map + def load_readme_config(self, readme_file): + for readme_path in self._readme_paths: + if resolve_path_to_uri(readme_path) == readme_file: + return parse_readme_file(readme_path)['config'] + return None + @property def tags(self): if self._tags is None: @@ -82,44 +88,45 @@ def tags(self): def _parse_readme_input_file_tags(self): tags = {} - if self._readme_path is None: + if not self._readme_paths: return tags - with open(self._readme_path, 'r', encoding='utf-8') as f: - readme = f.read() - - re_yaml = re.compile( - r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n', - flags=re.MULTILINE) - for piece in re_yaml.finditer(readme): - flags = piece[1] - yaml_body = piece[3] - if 'input-file' not in yaml_body: - continue + for readme_path in self._readme_paths: + with open(readme_path, 'r', encoding='utf-8') as f: + readme = f.read() + + re_yaml = re.compile( + r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n', + flags=re.MULTILINE) + for piece in re_yaml.finditer(readme): + flags = piece[1] + yaml_body = piece[3] + if 'input-file' not in yaml_body: + continue - try: - body = yaml.safe_load(yaml_body) - except yaml.YAMLError as err: - logger.error(f'ParseYamlFailed: {self} : {self._readme_path} {flags}: {err}') - continue - - files = [] - for file_path in body['input-file']: - file_path = file_path.replace('$(this-folder)/', '') - file_path = os.path.join(os.path.dirname(self._readme_path), *file_path.split('/')) - if not os.path.isfile(file_path): - logger.warning(f'FileNotExist: {self} : {file_path}') + try: + body = yaml.safe_load(yaml_body) + except yaml.YAMLError as err: + logger.error(f'ParseYamlFailed: {self} : {readme_path} {flags}: {err}') continue - files.append(file_path) - - if len(files): - tag = piece[2] - if tag is None: - tag = '' - tag = OpenAPIResourceProviderTag(tag.strip(), self) - if tag not in tags: - tags[tag] = set() - tags[tag] = tags[tag].union(files) + + files = [] + for file_path in body['input-file']: + file_path = file_path.replace('$(this-folder)/', '') + file_path = os.path.join(os.path.dirname(readme_path), *file_path.split('/')) + if not os.path.isfile(file_path): + logger.warning(f'FileNotExist: {self} : {file_path}') + continue + files.append(file_path) + + if len(files): + tag = piece[2] + if tag is None: + tag = '' + tag = OpenAPIResourceProviderTag(tag.strip(), self) + if tag not in tags: + tags[tag] = set() + tags[tag] = tags[tag].union(files) tags = [*tags.items()] tags.sort(key=lambda item: item[0].date, reverse=True) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_module.py b/src/aaz_dev/swagger/model/specs/_swagger_module.py index 66007747..76ff8e8b 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_module.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_module.py @@ -1,7 +1,8 @@ import os +import glob from utils.plane import PlaneEnum -from utils.config import Config +from swagger.utils.tools import resolve_path_to_uri from ._resource_provider import OpenAPIResourceProvider, TypeSpecResourceProvider from ._typespec_helper import TypeSpecHelper @@ -36,17 +37,6 @@ def names(self): return [self.name] else: return [*self._parent.names, self.name] - - @staticmethod - def resolve_path_to_uri(path): - relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/') - if relative_path.startswith('../'): - raise ValueError(f"Invalid path: {path}") - if relative_path.startswith('./'): - relative_path = relative_path[2:] - if relative_path.startswith('/'): - relative_path = relative_path[1:] - return relative_path class MgmtPlaneModule(SwaggerModule): @@ -77,8 +67,8 @@ def _get_openapi_resource_providers(self): if os.path.isdir(path): name_parts = name.split('.') if len(name_parts) >= 2: - readme_path = _search_readme_md_path(path, search_parent=True) - rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self)) + readme_paths = [*_search_readme_md_paths(path, search_parent=True)] + rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self)) elif name.lower() != 'common': # azsadmin module only sub_module = MgmtPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self) @@ -91,7 +81,7 @@ def _get_typespec_resource_providers(self): return rp for namespace, ts_path, cfg_path in TypeSpecHelper.find_mgmt_plane_entry_files(self.folder_path): - entry_file = self.resolve_path_to_uri(ts_path) + entry_file = resolve_path_to_uri(ts_path) if namespace in rp: rp[namespace].entry_files.append(entry_file) else: @@ -133,8 +123,8 @@ def _get_openapi_resource_providers(self): continue name_parts = name.split('.') if len(name_parts) >= 2: - readme_path = _search_readme_md_path(path, search_parent=True) - rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self)) + readme_paths = [*_search_readme_md_paths(path, search_parent=True)] + rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self)) elif name.lower() != 'common': sub_module = DataPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self) rp.extend(sub_module.get_resource_providers()) @@ -146,7 +136,7 @@ def _get_typespec_resource_providers(self): return rp for namespace, ts_path, cfg_path in TypeSpecHelper.find_data_plane_entry_files(self.folder_path): - entry_file = self.resolve_path_to_uri(ts_path) + entry_file = resolve_path_to_uri(ts_path) if namespace in rp: rp[namespace].entry_files.append(entry_file) else: @@ -155,21 +145,13 @@ def _get_typespec_resource_providers(self): return [*rp.values()] -def _search_readme_md_path(path, search_parent=False): +def _search_readme_md_paths(path, search_parent=False): + # Check parent directory first if requested if search_parent: - readme_path = os.path.join(os.path.dirname(path), 'readme.md') - if os.path.exists(readme_path): - return readme_path - - readme_path = os.path.join(path, 'readme.md') - if os.path.exists(readme_path): - return readme_path - - # find in sub directory - for name in os.listdir(path): - sub_path = os.path.join(path, name) - if os.path.isdir(sub_path): - readme_path = _search_readme_md_path(sub_path) - if readme_path is not None: - return readme_path - return None + parent_readme = os.path.join(os.path.dirname(path), 'readme.md') + if os.path.isfile(parent_readme): + yield parent_readme + + # Use glob to recursively find all readme.md files + pattern = os.path.join(path, '**', 'readme.md') + yield from glob.iglob(pattern, recursive=True) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_specs.py b/src/aaz_dev/swagger/model/specs/_swagger_specs.py index 57492f06..688d084e 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_specs.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_specs.py @@ -12,12 +12,12 @@ def __init__(self, folder_path): self._folder_path = folder_path @property - def _spec_folder_path(self): + def spec_folder_path(self): return os.path.join(self._folder_path, 'specification') def get_mgmt_plane_modules(self, plane): modules = [] - for name in os.listdir(self._spec_folder_path): + for name in os.listdir(self.spec_folder_path): module = self.get_mgmt_plane_module(name, plane=plane) if module: modules.append(module) @@ -30,7 +30,7 @@ def get_mgmt_plane_module(self, *names, plane): if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name): return None - path = os.path.join(self._spec_folder_path, name) + path = os.path.join(self.spec_folder_path, name) if not os.path.isdir(path): return None if os.path.isdir(os.path.join(path, 'resource-manager')) or TypeSpecHelper.find_mgmt_plane_entry_files(path): @@ -45,7 +45,7 @@ def get_mgmt_plane_module(self, *names, plane): def get_data_plane_modules(self, plane): modules = [] - for name in os.listdir(self._spec_folder_path): + for name in os.listdir(self.spec_folder_path): module = self.get_data_plane_module(name, plane=plane) if module: modules.append(module) @@ -58,7 +58,7 @@ def get_data_plane_module(self, *names, plane): if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name): return None - path = os.path.join(self._spec_folder_path, name) + path = os.path.join(self.spec_folder_path, name) if os.path.isdir(os.path.join(path, 'data-plane')) or TypeSpecHelper.find_data_plane_entry_files(path): module = DataPlaneModule(plane=plane, name=name, folder_path=path) for name in names[1:]: diff --git a/src/aaz_dev/swagger/utils/tools.py b/src/aaz_dev/swagger/utils/tools.py index 62f4c439..f2610ae8 100644 --- a/src/aaz_dev/swagger/utils/tools.py +++ b/src/aaz_dev/swagger/utils/tools.py @@ -4,6 +4,8 @@ # license information. # ----------------------------------------------------------------------------- import re +import os +from utils.config import Config URL_PARAMETER_PLACEHOLDER = "{}" @@ -39,3 +41,14 @@ def swagger_resource_path_to_resource_id(path): idx += 1 path_parts[0] = "/".join(url_parts).lower() return "?".join(path_parts) + + +def resolve_path_to_uri(path): + relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/') + if relative_path.startswith('../'): + raise ValueError(f"Invalid path: {path}") + if relative_path.startswith('./'): + relative_path = relative_path[2:] + if relative_path.startswith('/'): + relative_path = relative_path[1:] + return relative_path diff --git a/src/aaz_dev/utils/readme_helper.py b/src/aaz_dev/utils/readme_helper.py new file mode 100644 index 00000000..e9d24933 --- /dev/null +++ b/src/aaz_dev/utils/readme_helper.py @@ -0,0 +1,50 @@ +import yaml + +def _update_config(config, yaml_content): + for key, value in yaml_content.items(): + if key not in config: + config[key] = value + continue + if isinstance(value, dict): + _update_config(config[key], value) + elif isinstance(value, list): + config[key].extend(value) + else: + config[key] = value + +def parse_readme_file(readme_path: str): + """Parse the readme file title and combine basic config in the yaml section.""" + readme_config = {} + with open(readme_path, 'r', encoding='utf-8') as f: + content = f.readlines() + # content.append("```") # append a fake yaml section to make sure the last yaml section is ended + readme_title = None + in_yaml_section = False + yaml_content = [] + for line in content: + if not readme_title and line.strip().startswith("# ") and not in_yaml_section: + readme_title = line.strip()[2:].strip() + if line.strip().startswith("```") and 'yaml' in line: + condition = line.split('yaml')[1].strip() + # Do not parse the yaml section if it has the condition + if not condition: + in_yaml_section = True + elif in_yaml_section: + if line.strip().startswith("```"): + try: + yaml_config = yaml.load("\n".join(yaml_content), Loader=yaml.FullLoader) + except Exception as e: + raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_path}") + _update_config(readme_config, yaml_config) + in_yaml_section = False + yaml_content = [] + else: + if line.strip(): + yaml_content.append(line) + else: + yaml_content.append("") + + return { + "title": readme_title, + "config": readme_config + } From 556564ac33781c13b65b6533053f6ae33ff0469c Mon Sep 17 00:00:00 2001 From: kai ru <69238381+kairu-ms@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:09:10 +0800 Subject: [PATCH 07/18] Update src/aaz_dev/utils/readme_helper.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/aaz_dev/utils/readme_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/utils/readme_helper.py b/src/aaz_dev/utils/readme_helper.py index e9d24933..dd8a2887 100644 --- a/src/aaz_dev/utils/readme_helper.py +++ b/src/aaz_dev/utils/readme_helper.py @@ -32,7 +32,7 @@ def parse_readme_file(readme_path: str): elif in_yaml_section: if line.strip().startswith("```"): try: - yaml_config = yaml.load("\n".join(yaml_content), Loader=yaml.FullLoader) + yaml_config = yaml.safe_load("\n".join(yaml_content)) except Exception as e: raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_path}") _update_config(readme_config, yaml_config) From b9aa44421fe16701d4148bfd176dfd1151f489d4 Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 12:36:18 +0800 Subject: [PATCH 08/18] fix swagger tag yaml parse --- src/aaz_dev/swagger/model/specs/_resource_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index a2b91dfd..ce925954 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -96,7 +96,7 @@ def _parse_readme_input_file_tags(self): readme = f.read() re_yaml = re.compile( - r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n', + r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)\s*```\s*\n', flags=re.MULTILINE) for piece in re_yaml.finditer(readme): flags = piece[1] From 08f377433b96c45ceb7445d39daaef3df3c7f3d0 Mon Sep 17 00:00:00 2001 From: kai ru Date: Mon, 16 Dec 2024 11:49:29 +0800 Subject: [PATCH 09/18] init ps module --- src/aaz_dev/app/app.py | 4 ++++ src/aaz_dev/ps/__init__.py | 0 src/aaz_dev/ps/api/__init__.py | 5 +++++ src/aaz_dev/ps/api/_cmds.py | 11 +++++++++++ src/aaz_dev/ps/api/autorest.py | 16 ++++++++++++++++ src/aaz_dev/ps/controller/__init__.py | 0 src/aaz_dev/ps/model/__init__.py | 0 src/aaz_dev/ps/templates/__init__.py | 14 ++++++++++++++ src/aaz_dev/ps/templates/_filters.py | 6 ++++++ src/aaz_dev/ps/tests/__init__.py | 0 10 files changed, 56 insertions(+) create mode 100644 src/aaz_dev/ps/__init__.py create mode 100644 src/aaz_dev/ps/api/__init__.py create mode 100644 src/aaz_dev/ps/api/_cmds.py create mode 100644 src/aaz_dev/ps/api/autorest.py create mode 100644 src/aaz_dev/ps/controller/__init__.py create mode 100644 src/aaz_dev/ps/model/__init__.py create mode 100644 src/aaz_dev/ps/templates/__init__.py create mode 100644 src/aaz_dev/ps/templates/_filters.py create mode 100644 src/aaz_dev/ps/tests/__init__.py diff --git a/src/aaz_dev/app/app.py b/src/aaz_dev/app/app.py index 70f20352..90db0d45 100644 --- a/src/aaz_dev/app/app.py +++ b/src/aaz_dev/app/app.py @@ -46,6 +46,10 @@ def invalid_api_usage(e): from cli.api import register_blueprints register_blueprints(app) + # register routes of ps module + from ps.api import register_blueprints + register_blueprints(app) + return app diff --git a/src/aaz_dev/ps/__init__.py b/src/aaz_dev/ps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/aaz_dev/ps/api/__init__.py b/src/aaz_dev/ps/api/__init__.py new file mode 100644 index 00000000..788da3b5 --- /dev/null +++ b/src/aaz_dev/ps/api/__init__.py @@ -0,0 +1,5 @@ + +def register_blueprints(app): + from . import _cmds, autorest + app.register_blueprint(_cmds.bp) + app.register_blueprint(autorest.bp) diff --git a/src/aaz_dev/ps/api/_cmds.py b/src/aaz_dev/ps/api/_cmds.py new file mode 100644 index 00000000..7cf64cda --- /dev/null +++ b/src/aaz_dev/ps/api/_cmds.py @@ -0,0 +1,11 @@ +import click +import logging +from flask import Blueprint +import sys + +from utils.config import Config + +logger = logging.getLogger('backend') + +bp = Blueprint('ps-cmds', __name__, url_prefix='/PS/CMDs', cli_group="ps") +bp.cli.short_help = "Manage powershell commands." diff --git a/src/aaz_dev/ps/api/autorest.py b/src/aaz_dev/ps/api/autorest.py new file mode 100644 index 00000000..e91d663b --- /dev/null +++ b/src/aaz_dev/ps/api/autorest.py @@ -0,0 +1,16 @@ +from flask import Blueprint, jsonify, request, url_for + +from utils.config import Config +from utils import exceptions +from command.controller.specs_manager import AAZSpecsManager +import logging + +logging.basicConfig(level="INFO") + + +bp = Blueprint('autorest', __name__, url_prefix='/PS/Autorest') + + +@bp.route("/Directives", methods=("GET", )) +def az_profiles(): + return jsonify({"test": "test"}) diff --git a/src/aaz_dev/ps/controller/__init__.py b/src/aaz_dev/ps/controller/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/aaz_dev/ps/model/__init__.py b/src/aaz_dev/ps/model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/aaz_dev/ps/templates/__init__.py b/src/aaz_dev/ps/templates/__init__.py new file mode 100644 index 00000000..d42d0041 --- /dev/null +++ b/src/aaz_dev/ps/templates/__init__.py @@ -0,0 +1,14 @@ +_templates = None + + +def get_templates(): + global _templates + if _templates is None: + import os + from jinja2 import Environment, FileSystemLoader + from ._filters import custom_filters + env = Environment(loader=FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(__file__)))) + env.filters.update(custom_filters) + _templates = { + } + return _templates diff --git a/src/aaz_dev/ps/templates/_filters.py b/src/aaz_dev/ps/templates/_filters.py new file mode 100644 index 00000000..f9d8975a --- /dev/null +++ b/src/aaz_dev/ps/templates/_filters.py @@ -0,0 +1,6 @@ + + +custom_filters = { + # "camel_case": camel_case, + # "snake_case": snake_case, +} diff --git a/src/aaz_dev/ps/tests/__init__.py b/src/aaz_dev/ps/tests/__init__.py new file mode 100644 index 00000000..e69de29b From f793f05bc140907f11ca7377781c816e4c3f4bb5 Mon Sep 17 00:00:00 2001 From: kai ru Date: Mon, 16 Dec 2024 13:49:59 +0800 Subject: [PATCH 10/18] add default tag for resource provider --- .../swagger/model/specs/_resource_provider.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index ce925954..d98ba907 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -80,6 +80,29 @@ def load_readme_config(self, readme_file): return parse_readme_file(readme_path)['config'] return None + @property + def default_tag(self): + if self._readme_path is None: + return None + + with open(self._readme_path, 'r', encoding='utf-8') as f: + readme = f.read() + lines = readme.split('\n') + for i in range(len(lines)): + line = lines[i] + if line.startswith('### Basic Information'): + lines = lines[i+1:] + break + latest_tag = None + for i in range(len(lines)): + line = lines[i] + if line.startswith('##'): + break + if line.startswith('tag:'): + latest_tag = line.split(':')[-1].strip() + break + return latest_tag + @property def tags(self): if self._tags is None: From 7744db15c101b744aa9f59f68ff2c377d25dde42 Mon Sep 17 00:00:00 2001 From: kai ru Date: Mon, 16 Dec 2024 14:23:05 +0800 Subject: [PATCH 11/18] Move demo code --- src/aaz_dev/ps/api/_cmds.py | 129 ++++ .../autorest_configuration_generator.py | 616 ++++++++++++++++++ src/aaz_dev/ps/templates/__init__.py | 3 + .../templates/autorest/configuration.yaml.j2 | 53 ++ 4 files changed, 801 insertions(+) create mode 100644 src/aaz_dev/ps/controller/autorest_configuration_generator.py create mode 100644 src/aaz_dev/ps/templates/autorest/configuration.yaml.j2 diff --git a/src/aaz_dev/ps/api/_cmds.py b/src/aaz_dev/ps/api/_cmds.py index 7cf64cda..c8975dd2 100644 --- a/src/aaz_dev/ps/api/_cmds.py +++ b/src/aaz_dev/ps/api/_cmds.py @@ -2,6 +2,8 @@ import logging from flask import Blueprint import sys +import os +import subprocess from utils.config import Config @@ -9,3 +11,130 @@ bp = Blueprint('ps-cmds', __name__, url_prefix='/PS/CMDs', cli_group="ps") bp.cli.short_help = "Manage powershell commands." + + +@bp.cli.command("generate-powershell", short_help="Generate powershell code based on selected azure cli module.") +@click.option( + "--aaz-path", '-a', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + default=Config.AAZ_PATH, + required=not Config.AAZ_PATH, + callback=Config.validate_and_setup_aaz_path, + expose_value=False, + help="The local path of aaz repo." +) +@click.option( + "--cli-path", '-c', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + callback=Config.validate_and_setup_cli_path, + help="The local path of azure-cli repo. Only required when generate from azure-cli module." +) +@click.option( + "--cli-extension-path", '-e', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + callback=Config.validate_and_setup_cli_extension_path, + help="The local path of azure-cli-extension repo. Only required when generate from azure-cli extension." +) +@click.option( + "--powershell-path", '-p', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + required=True, + help="The local path of azure-powershell repo." +) +@click.option( + "--extension-or-module-name", '--name', + required=True, + help="Name of the module in azure-cli or the extension in azure-cli-extensions" +) +@click.option( + "--swagger-path", '-s', + type=click.Path(file_okay=False, dir_okay=True, readable=True, resolve_path=True), + default=Config.SWAGGER_PATH, + required=not Config.SWAGGER_PATH, + callback=Config.validate_and_setup_swagger_path, + expose_value=False, + help="The local path of azure-rest-api-specs repo. Official repo is https://github.com/Azure/azure-rest-api-specs" +) +def generate_powershell(extension_or_module_name, cli_path=None, cli_extension_path=None, powershell_path=None): + from ps.controller.autorest_configuration_generator import PSAutoRestConfigurationGenerator + from cli.controller.az_module_manager import AzMainManager, AzExtensionManager + from ps.templates import get_templates + + # Module path in azure-powershell repo + + powershell_path = os.path.join(powershell_path, "src") + if not os.path.exists(powershell_path): + logger.error(f"Path `{powershell_path}` not exist") + sys.exit(1) + + if cli_path is not None: + assert Config.CLI_PATH is not None + manager = AzMainManager() + else: + assert cli_extension_path is not None + assert Config.CLI_EXTENSION_PATH is not None + manager = AzExtensionManager() + + if not manager.has_module(extension_or_module_name): + logger.error(f"Cannot find module or extension `{extension_or_module_name}`") + sys.exit(1) + + # generate README.md for powershell from CLI, ex, for Oracle, README.md should be generated in src/Oracle/Oracle.Autorest/README.md in azure-powershell repo + ps_generator = PSAutoRestConfigurationGenerator(manager, extension_or_module_name) + ps_cfg = ps_generator.generate_config() + + autorest_module_path = os.path.join(powershell_path, ps_cfg.module_name, f"{ps_cfg.module_name}.Autorest") + if not os.path.exists(autorest_module_path): + os.makedirs(autorest_module_path) + readme_file = os.path.join(autorest_module_path, "README.md") + if os.path.exists(readme_file): + # read until to the "### AutoRest Configuration" + with open(readme_file, "r") as f: + lines = f.readlines() + for i, line in enumerate(lines): + if line.startswith("### AutoRest Configuration"): + lines = lines[:i] + break + else: + lines = [] + + tmpl = get_templates()['autorest']['configuration'] + data = tmpl.render(cfg=ps_cfg) + lines.append(data) + with open(readme_file, "w") as f: + f.writelines(lines) + + print(f"Generated {readme_file}") + # Generate and build PowerShell module from the README.md file generated above + print("Start to generate the PowerShell module from the README.md file in " + autorest_module_path) + + # Execute autorest to generate the PowerShell module + original_cwd = os.getcwd() + os.chdir(autorest_module_path) + exit_code = os.system("pwsh -Command autorest") + + # Print the output of the generation + if (exit_code != 0): + print("Failed to generate the module") + os.chdir(original_cwd) + sys.exit(1) + else: + print("Code generation succeeded.") + # print(result.stdout) + + os.chdir(original_cwd) + # Execute autorest to generate the PowerShell module + print("Start to build the generated PowerShell module") + result = subprocess.run( + ["pwsh", "-File", 'build-module.ps1'], + capture_output=True, + text=True, + cwd=autorest_module_path + ) + + if (result.returncode != 0): + print("Failed to build the module, please see following output for details:") + print(result.stderr) + sys.exit(1) + else: + print("Module build succeeds, and you may run the generated module by executing the following command: `./run-module.ps1` in " + autorest_module_path) diff --git a/src/aaz_dev/ps/controller/autorest_configuration_generator.py b/src/aaz_dev/ps/controller/autorest_configuration_generator.py new file mode 100644 index 00000000..0d18f862 --- /dev/null +++ b/src/aaz_dev/ps/controller/autorest_configuration_generator.py @@ -0,0 +1,616 @@ +from cli.controller.az_module_manager import AzModuleManager +from swagger.controller.specs_manager import SwaggerSpecsManager +from command.controller.specs_manager import AAZSpecsManager +from utils.config import Config +from utils.exceptions import ResourceNotFind +from command.model.configuration import CMDConfiguration, CMDHttpOperation +import logging +from pluralizer import Pluralizer +import re +import os +from swagger.model.specs._utils import map_path_2_repo + +from fuzzywuzzy import fuzz + + +logger = logging.getLogger("backend") + +class PSAutoRestConfiguration: + + def __init__(self): + self.commit = None + self.version = None + self.module_name = None + self.readme_file = None + self.removed_subjects = [] + self.removed_verbs = [] + + +class PSAutoRestConfigurationGenerator: + _CAMEL_CASE_PATTERN = re.compile(r"^([a-zA-Z][a-z0-9]+)(([A-Z][a-z0-9]*)+)$") + _pluralizer = Pluralizer() + + def __init__(self, az_module_manager: AzModuleManager, module_name) -> None: + self.module_manager = az_module_manager + self.module_name = module_name + self.aaz_specs_manager = AAZSpecsManager() + self.swagger_specs_manager = SwaggerSpecsManager() + self._ps_profile = None + + def generate_config(self): + ps_cfg = PSAutoRestConfiguration() + # TODO: get the commit using git + ps_cfg.commit = "cbbe228fd422db02b65e2748f83df5f2bcad7581" + + module = self.module_manager.load_module(self.module_name) + + cli_profile = {} + + for cli_command in self.iter_cli_commands( + module.profiles[Config.CLI_DEFAULT_PROFILE] + ): + names = cli_command.names + version_name = cli_command.version + aaz_cmd = self.aaz_specs_manager.find_command(*names) + if not aaz_cmd: + raise ResourceNotFind( + "Command '{}' not exist in AAZ".format(" ".join(names)) + ) + version = None + for v in aaz_cmd.versions or []: + if v.name == version_name: + version = v + break + if not version: + raise ResourceNotFind( + "Version '{}' of command '{}' not exist in AAZ".format( + version_name, " ".join(names) + ) + ) + resource = v.resources[0] + cfg: CMDConfiguration = self.aaz_specs_manager.load_resource_cfg_reader( + resource.plane, resource.id, resource.version + ) + if not cfg: + raise ResourceNotFind( + "Resource Configuration '{}' not exist in AAZ".format(resource.id) + ) + for resource in cfg.resources: + tag = (resource.plane, "/".join(resource.mod_names), resource.rp_name) + if tag not in cli_profile: + cli_profile[tag] = {} + if resource.id not in cli_profile[tag]: + cli_profile[tag][resource.id] = { + "path": resource.path, + "cfg": cfg, + "commands": [], + "subresources": [], + } + cli_profile[tag][resource.id]["commands"].append(cli_command.names) + if resource.subresource: + cli_profile[tag][resource.id]["subresources"].append( + resource.subresource + ) + + # TODO: let LLM to choice the plane and rp_name later + if len(cli_profile.keys()) > 1: + raise ValueError("Only one plane module and rp_name is supported") + (plane, mod_names, rp_name) = list(cli_profile.keys())[0] + cli_resources = cli_profile[(plane, mod_names, rp_name)] + + module_manager = self.swagger_specs_manager.get_module_manager( + plane, mod_names.split("/") + ) + rp = module_manager.get_openapi_resource_provider(rp_name) + swagger_resources = rp.get_resource_map_by_tag(rp.default_tag) + if not swagger_resources: + raise ResourceNotFind("Resources not find in Swagger") + + readme_parts= rp._readme_path.split(os.sep) + ps_cfg.readme_file = '/'.join(readme_parts[readme_parts.index("specification"):]) + ps_cfg.version = "0.1.0" + ps_cfg.module_name = mod_names.split("/")[0] + ps_cfg.module_name = ps_cfg.module_name[0].upper() + ps_cfg.module_name[1:] + + ps_profile = {} + + # create ps_profile by swagger resources + for resource_id, resource in swagger_resources.items(): + resource = list(resource.values())[0] + methods = set() + operations = {} + op_group_name = self.get_operation_group_name(resource) + if resource_id not in cli_resources: + # the whole resource id is not used in cli + for op_tag, method in resource.operations.items(): + operations[op_tag] = { + "tag": op_tag, + "delete": True, + "resource_id": resource_id, + "method": method, + } + methods.add(method) + else: + for cmd_names in cli_resources[resource_id]["commands"]: + cfg = cli_resources[resource_id]["cfg"] + command = cfg.find_command(*cmd_names) + assert command is not None + for cmd_op in command.operations: + if not isinstance(cmd_op, CMDHttpOperation): + continue + op_tag = cmd_op.operation_id + if op_tag not in resource.operations: + # make sure the operation is from the same resource + continue + method = resource.operations[op_tag] + operations[op_tag] = { + "tag": op_tag, + "delete": False, + "resource_id": resource_id, + "method": method, + } + methods.add(method) + + for op_tag, method in resource.operations.items(): + if op_tag not in operations: + operations[op_tag] = { + "tag": op_tag, + "delete": ( + False if method == "patch" else True + ), # PowerShell Prefer Patch Method for update commands + "resource_id": resource_id, + "method": method, + } + methods.add(method) + + for op_tag, op in operations.items(): + variants = self.inferCommandNames(op_tag, op_group_name) + op['variants'] = [] + for variant in variants: + if op['method'] == 'put' and variant['action'] == 'Update': + if 'get' not in methods: + # "update" should be "set" if it's a PUT and not the generic update (GET+PUT) + variant['verb'] = 'Set' + else: + use_generic_update = True + for patch_op in operations.values(): + if patch_op['method'] == 'patch': + use_generic_update = patch_op['delete'] + break + if not use_generic_update: + continue + op['variants'].append(variant) + # make sure the variants should have the same subject + subjects = list(set([v['subject'] for v in op['variants']])) + assert len(subjects) == 1, f"Operation {op_tag} has different subjects: {subjects}" + + subject = subjects[0] + if subject not in ps_profile: + ps_profile[subject] = { + "operations": {}, + "delete": False, + } + ps_profile[subject]["operations"][op_tag] = op + + for subject in ps_profile.values(): + subject_delete = True + for op in subject['operations'].values(): + if not op['delete']: + subject_delete = False + break + subject['delete'] = subject_delete + + self._ps_profile = ps_profile + + ps_cfg.removed_subjects = [] + ps_cfg.removed_verbs = [] + for subject_name, subject in ps_profile.items(): + if subject['delete']: + ps_cfg.removed_subjects.append(subject_name) + continue + for op in subject['operations'].values(): + if op['delete']: + for variant in op['variants']: + ps_cfg.removed_verbs.append((variant['subject'], variant['verb'])) + return ps_cfg + + def iter_cli_commands(self, profile): + for command_group in profile.command_groups.values(): + for cli_command in self._iter_cli_commands(command_group): + yield cli_command + + def _iter_cli_commands(self, view_command_group): + if view_command_group.commands: + for cli_command in view_command_group.commands.values(): + yield cli_command + if view_command_group.command_groups: + for command_group in view_command_group.command_groups.values(): + for cli_command in self._iter_cli_commands(command_group): + yield cli_command + + def get_operation_group_name(self, resource): + operation_groups = set() + for operation_id, method in resource.operations.items(): + op_group = self._parse_operation_group_name(resource, operation_id, method) + operation_groups.add(op_group) + + if None in operation_groups: + return None + + if len(operation_groups) == 1: + return operation_groups.pop() + + op_group_name = sorted( + operation_groups, + key=lambda nm: fuzz.partial_ratio( + resource.id, nm + ), # use the name which is closest to resource_id + reverse=True, + )[0] + return op_group_name + + def _parse_operation_group_name(self, resource, op_id, method): + # extract operation group name from operation_id + value = op_id.strip() + value = value.replace("-", "_") + if "_" in value: + parts = value.split("_") + op_group_name = parts[0] + if op_group_name.lower() in ("create", "get", "update", "delete", "patch"): + op_group_name = parts[1] + else: + if " " in value: + value = value.replace(" ", "") # Changed to Camel Case + match = self._CAMEL_CASE_PATTERN.match(value) + if not match: + logger.error( + f"InvalidOperationIdFormat:" + f"\toperationId should be in format of '[OperationGroupName]_[OperationName]' " + f"or '[Verb][OperationGroupName]':\n" + f"\tfile: {map_path_2_repo(resource.file_path)}\n" + f"\tpath: {resource.path}\n" + f"\tmethod: {method} operationId: {op_id}\n" + ) + return None + op_group_name = match[2] # [OperationGroupName] + + return self.singular_noun(op_group_name) + + @classmethod + def inferCommandNames(cls, operation_id, op_group_name): + parts = operation_id.split("_") + assert len(parts) == 2 + method = parts[1] + method = method[0].upper() + method[1:] + + if VERB_MAPPING.get(method): + return [ + cls.create_command_variant(method, [op_group_name], []) + ] + + # split camel case to words + words = re.findall(r'[A-Z][a-z]*', method) + return cls._infer_command(words, op_group_name, []) + + @classmethod + def _infer_command(cls, operation, op_group_name, suffix): + operation = [w for w in operation if w != 'All'] + if len(operation) == 1: + # simple operation, just an id with a single value + return [ + cls.create_command_variant(operation[0], [op_group_name], suffix) + ] + + if len(operation) == 2: + # should try to infer [SUBJECT] and [ACTION] from operation + if VERB_MAPPING.get(operation[0]): + # [ACTION][SUBJECT] + return [ + cls.create_command_variant(operation[0], [op_group_name, operation[1]], suffix) + ] + if VERB_MAPPING.get(operation[1]): + # [SUBJECT][ACTION] + return [ + cls.create_command_variant(operation[1], [op_group_name, operation[0]], suffix) + ] + logger.warning(f"Operation ${operation[0]}/${operation[1]} is inferred without finding action.") + return [ + cls.create_command_variant(operation[0], [op_group_name, operation[1]], suffix) + ] + + # three or more words. + # first, see if it's an 'or' + if 'Or' in operation: + idx = operation.index('Or') + return cls._infer_command( + operation[:idx] + operation[idx+2:], + op_group_name, + suffix + ) + cls._infer_command( + operation[idx+1:], + op_group_name, + suffix + ) + + for w in ['With', 'At', 'By', 'For', 'In', 'Of']: + if w in operation: + idx = operation.index(w) + if idx > 0: + # so this is something like DoActionWithStyle + return cls._infer_command( + operation[:idx], + op_group_name, + operation[idx:], + ) + + # if not, then seek out a verb from there. + for i in range(len(operation)): + if VERB_MAPPING.get(operation[i]): + # if the action is first + if i == 0: + # everything else is the subject + return [ + cls.create_command_variant(operation[0], [op_group_name] + operation[1:], suffix) + ] + if i == len(operation) - 1: + # if it's last, the subject would be the first thing + return [ + cls.create_command_variant(operation[i], [op_group_name] + operation[:i], suffix) + ] + # otherwise + # things before are part of the subject + # things after the verb should be part of the suffix + return [ + cls.create_command_variant(operation[i], [op_group_name] + operation[:i], suffix + operation[i+1:]) + ] + + # so couldn't tell what the action was. + # fallback to the original behavior with a warning. + logger.warning(f"Operation ${operation[0]}/${operation[1]} is inferred without finding action.") + return [ + cls.create_command_variant(operation[0], [op_group_name] + operation[1:], suffix) + ] + + @classmethod + def create_command_variant(cls, action, subject, variant): + verb = cls.get_powershell_verb(action) + if verb == 'Invoke': + # if the 'operation' name was "post" -- it's kindof redundant. + # so, only include the operation name in the group name if it's anything else + if action.lower() != 'post': + subject = [action] + subject + subject = [cls.singular_noun(s) or s for s in subject] + # remove duplicate values + values = set() + simplified_subject = [] + for s in subject: + if s in values: + continue + values.add(s) + simplified_subject.append(s) + return { + "subject": ''.join(simplified_subject), + "verb": verb, + "variant": ''.join(variant), + "action": action, + } + + @staticmethod + def get_powershell_verb(action): + action = action[0].upper() + action[1:].lower() + return VERB_MAPPING.get(action, "Invoke") + + @classmethod + def singular_noun(cls, noun): + words = re.findall(r'[A-Z][a-z]*', noun) + # singular the last word in operation group name + w = cls._pluralizer.singular(words[-1].lower()) + words[-1] = w[0].upper() + w[1:] + noun = ''.join(words) + return noun + + +VERB_MAPPING = { + "Access": "Get", + "Acquire": "Get", + "Activate": "Initialize", + "Add": "Add", + "Allocate": "New", + "Analyze": "Test", + "Append": "Add", + "Apply": "Add", + "Approve": "Approve", + "Assert": "Assert", + "Assign": "Set", + "Associate": "Join", + "Attach": "Add", + "Authorize": "Grant", + "Backup": "Backup", + "Block": "Block", + "Build": "Build", + "Bypass": "Skip", + "Cancel": "Stop", + "Capture": "Export", + "Cat": "Get", + "Change": "Rename", + "Check": "Test", + "Checkpoint": "Checkpoint", + "Clear": "Clear", + "Clone": "Copy", + "Close": "Close", + "Combine": "Join", + "Compare": "Compare", + "Compile": "Build", + "Complete": "Complete", + "Compress": "Compress", + "Concatenate": "Add", + "Configure": "Set", + "Confirm": "Confirm", + "Connect": "Connect", + "Convert": "Convert", + "ConvertFrom": "ConvertFrom", + "ConvertTo": "ConvertTo", + "Copy": "Copy", + "Create": "New", + "Cut": "Remove", + "Debug": "Debug", + "Delete": "Remove", + "Deny": "Deny", + "Deploy": "Deploy", + "Dir": "Get", + "Disable": "Disable", + "Discard": "Remove", + "Disconnect": "Disconnect", + "Discover": "Find", + "Dismount": "Dismount", + "Display": "Show", + "Dispose": "Remove", + "Dump": "Get", + "Duplicate": "Copy", + "Edit": "Edit", + "Enable": "Enable", + "End": "Stop", + "Enter": "Enter", + "Erase": "Clear", + "Evaluate": "Test", + "Examine": "Get", + "Execute": "Invoke", + "Exit": "Exit", + "Expand": "Expand", + "Export": "Export", + "Failover": "Set", + "Find": "Find", + "Finish": "Complete", + "Flush": "Clear", + "ForceReboot": "Restart", + "Format": "Format", + "Generalize": "Reset", + "Generate": "New", + "Get": "Get", + "Grant": "Grant", + "Group": "Group", + "Hide": "Hide", + "Import": "Import", + "Initialize": "Initialize", + "Insert": "Add", + "Install": "Install", + "Into": "Enter", + "Invoke": "Invoke", + "Is": "Test", + "Join": "Join", + "Jump": "Skip", + "Limit": "Limit", + "List": "Get", + "Load": "Import", + "Locate": "Find", + "Lock": "Lock", + "Make": "New", + "Measure": "Measure", + "Merge": "Merge", + "Migrate": "Move", + "Mount": "Mount", + "Move": "Move", + "Name": "Move", + "New": "New", + "Notify": "Send", + "Nullify": "Clear", + "Obtain": "Get", + "Open": "Open", + "Optimize": "Optimize", + "Out": "Out", + "Patch": "Update", + "Pause": "Suspend", + "Perform": "Invoke", + "Ping": "Ping", + "Pop": "Pop", + "Post": "Invoke", + "Power": "Start", + "PowerOff": "Stop", + "PowerOn": "Start", + "Produce": "Show", + "Protect": "Protect", + "Provision": "New", + "Publish": "Publish", + "Purge": "Clear", + "Push": "Push", + "Put": "Set", + "Read": "Read", + "Reassociate": "Move", + "Reboot": "Restart", + "Receive": "Receive", + "Recover": "Restore", + "Redo": "Redo", + "Refresh": "Update", + "Regenerate": "New", + "Register": "Register", + "Reimage": "Update", + "Release": "Publish", + "Remove": "Remove", + "Rename": "Rename", + "Repair": "Repair", + "Replace": "Update", + "Replicate": "Copy", + "Reprocess": "Update", + "Request": "Request", + "Reset": "Reset", + "Resize": "Resize", + "Resolve": "Resolve", + "Restart": "Restart", + "Restore": "Restore", + "Restrict": "Lock", + "Resubmit": "Submit", + "Resume": "Resume", + "Retarget": "Update", + "Retrieve": "Get", + "Revoke": "Revoke", + "Run": "Start", + "Save": "Save", + "Search": "Search", + "Secure": "Lock", + "Select": "Select", + "Send": "Send", + "Separate": "Split", + "Set": "Set", + "Show": "Show", + "Shutdown": "Stop", + "Skip": "Skip", + "Split": "Split", + "Start": "Start", + "Step": "Step", + "Stop": "Stop", + "Submit": "Submit", + "Suggest": "Get", + "Suspend": "Suspend", + "Swap": "Switch", + "Switch": "Switch", + "Sync": "Sync", + "Synch": "Sync", + "Synchronize": "Sync", + "Test": "Test", + "Trace": "Trace", + "Transfer": "Move", + "Trigger": "Start", + "Type": "Get", + "Unblock": "Unblock", + "Undelete": "Restore", + "Undo": "Undo", + "Uninstall": "Uninstall", + "Unite": "Join", + "Unlock": "Unlock", + "Unmark": "Clear", + "Unprotect": "Unprotect", + "Unpublish": "Unpublish", + "Unregister": "Unregister", + "Unrestrict": "Unlock", + "Unsecure": "Unlock", + "Unset": "Clear", + "Update": "Update", + "Upgrade": "Update", + "Use": "Use", + "Validate": "Test", + "Verify": "Test", + "Wait": "Wait", + "Watch": "Watch", + "Wipe": "Clear", + "Write": "Write", +} \ No newline at end of file diff --git a/src/aaz_dev/ps/templates/__init__.py b/src/aaz_dev/ps/templates/__init__.py index d42d0041..19603c83 100644 --- a/src/aaz_dev/ps/templates/__init__.py +++ b/src/aaz_dev/ps/templates/__init__.py @@ -10,5 +10,8 @@ def get_templates(): env = Environment(loader=FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(__file__)))) env.filters.update(custom_filters) _templates = { + "autorest": { + "configuration": env.get_template("autorest/configuration.yaml.j2"), + } } return _templates diff --git a/src/aaz_dev/ps/templates/autorest/configuration.yaml.j2 b/src/aaz_dev/ps/templates/autorest/configuration.yaml.j2 new file mode 100644 index 00000000..ef8084e0 --- /dev/null +++ b/src/aaz_dev/ps/templates/autorest/configuration.yaml.j2 @@ -0,0 +1,53 @@ +### AutoRest Configuration +> see https://aka.ms/autorest + +```yaml +commit: {{ cfg.commit }} +require: + - $(this-folder)/../../readme.azure.noprofile.md + - $(repo)/{{ cfg.readme_file }} + +try-require: + - $(repo)/{{ cfg.readme_file }} + +module-version: {{ cfg.version }} +title: {{ cfg.module_name }} +subject-prefix: $(service-name) + +inlining-threshold: 100 +resourcegroup-append: true +nested-object-to-string: true +identity-correction-for-post: true + +directive: + # Model complex objects + + # Remove the set-* cmdlet + - where: + verb: Set + remove: true + + # Remove APIs + {%- if cfg.removed_subjects|length %} + - where: + subject: {{ cfg.removed_subjects | join("|") }} + remove: true + {%- endif %} + {%- for subject, verb in cfg.removed_verbs %} + - where: + subject: {{ subject }} + verb: {{ verb }} + remove: true + {%- endfor %} + + # Remove variants + - where: + variant: ^(Create|Update)(?!.*?(Expanded|JsonFilePath|JsonString)) + remove: true + - where: + variant: ^CreateViaIdentity.*$ + remove: true + + # Rename parameter + +``` From 5b0bd909601a38911decda2de2dc15e206157547 Mon Sep 17 00:00:00 2001 From: kai ru Date: Wed, 18 Dec 2024 15:46:15 +0800 Subject: [PATCH 12/18] add argument, environment-variable and api for powershell folder path --- src/aaz_dev/app/run.py | 8 +++ src/aaz_dev/ps/api/__init__.py | 3 +- src/aaz_dev/ps/api/_cmds.py | 4 +- src/aaz_dev/ps/api/powershell.py | 24 +++++++ .../ps/controller/ps_module_manager.py | 17 +++++ .../swagger/model/specs/_swagger_specs.py | 72 ++++++++++++++++++- src/aaz_dev/utils/config.py | 13 ++++ 7 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 src/aaz_dev/ps/api/powershell.py create mode 100644 src/aaz_dev/ps/controller/ps_module_manager.py diff --git a/src/aaz_dev/app/run.py b/src/aaz_dev/app/run.py index 49895438..2bc02851 100644 --- a/src/aaz_dev/app/run.py +++ b/src/aaz_dev/app/run.py @@ -85,6 +85,14 @@ def is_port_in_use(host, port): expose_value=False, help="The local path of azure-cli-extension repo. Official repo is https://github.com/Azure/azure-cli-extensions" ) +@click.option( + "--powershell-path", '--ps', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + default=Config.POWERSHELL_PATH, + callback=Config.validate_and_setup_powershell_path, + expose_value=False, + help="The local path of azure-powershell repo." +) @click.option( "--workspaces-path", '-w', type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), diff --git a/src/aaz_dev/ps/api/__init__.py b/src/aaz_dev/ps/api/__init__.py index 788da3b5..b05934ac 100644 --- a/src/aaz_dev/ps/api/__init__.py +++ b/src/aaz_dev/ps/api/__init__.py @@ -1,5 +1,6 @@ def register_blueprints(app): - from . import _cmds, autorest + from . import _cmds, autorest, powershell app.register_blueprint(_cmds.bp) app.register_blueprint(autorest.bp) + app.register_blueprint(powershell.bp) diff --git a/src/aaz_dev/ps/api/_cmds.py b/src/aaz_dev/ps/api/_cmds.py index c8975dd2..5f498851 100644 --- a/src/aaz_dev/ps/api/_cmds.py +++ b/src/aaz_dev/ps/api/_cmds.py @@ -36,9 +36,9 @@ help="The local path of azure-cli-extension repo. Only required when generate from azure-cli extension." ) @click.option( - "--powershell-path", '-p', + "--powershell-path", '--ps', type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), - required=True, + callback=Config.validate_and_setup_powershell_path, help="The local path of azure-powershell repo." ) @click.option( diff --git a/src/aaz_dev/ps/api/powershell.py b/src/aaz_dev/ps/api/powershell.py new file mode 100644 index 00000000..4965699c --- /dev/null +++ b/src/aaz_dev/ps/api/powershell.py @@ -0,0 +1,24 @@ +from flask import Blueprint, jsonify, request, url_for + +from utils.config import Config +from utils import exceptions +from command.controller.specs_manager import AAZSpecsManager +import logging + +logging.basicConfig(level="INFO") + + +bp = Blueprint('powershell', __name__, url_prefix='/PS/Powershell') + + +@bp.route("/Path", methods=("GET", "PUT")) +def powershell_path(): + if request.method == "GET": + return jsonify({"path": Config.POWERSHELL_PATH}) + elif request.method == "PUT": + data = request.json + try: + Config.validate_and_setup_powershell_path(None, None, data["path"]) + except ValueError as e: + raise exceptions.InvalidAPIUsage(str(e)) + return jsonify({"path": Config.POWERSHELL_PATH}) diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py new file mode 100644 index 00000000..264b9e2b --- /dev/null +++ b/src/aaz_dev/ps/controller/ps_module_manager.py @@ -0,0 +1,17 @@ +import ast +import glob +import json +import logging +import os +import pkgutil +import re + +from utils import exceptions +from utils.config import Config +from collections import deque + +logger = logging.getLogger('backend') + + +class PSModuleManager: + pass diff --git a/src/aaz_dev/swagger/model/specs/_swagger_specs.py b/src/aaz_dev/swagger/model/specs/_swagger_specs.py index 688d084e..6b6e8dd4 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_specs.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_specs.py @@ -1,8 +1,9 @@ import os +import subprocess from utils.plane import PlaneEnum -from ._swagger_module import MgmtPlaneModule, DataPlaneModule from utils.exceptions import ResourceNotFind, InvalidAPIUsage +from ._swagger_module import MgmtPlaneModule, DataPlaneModule from ._typespec_helper import TypeSpecHelper @@ -10,6 +11,8 @@ class SwaggerSpecs: def __init__(self, folder_path): self._folder_path = folder_path + self._repo_name = self._get_repo_name() + self._remote_name = self._get_upstream_remote_name() @property def spec_folder_path(self): @@ -67,8 +70,75 @@ def get_data_plane_module(self, *names, plane): return None module = DataPlaneModule(plane=plane, name=name, folder_path=path, parent=module) return module + return None + + def _get_repo_name(self): + # get repo name from origin remote + git_config_path = os.path.join(self._folder_path, '.git', 'config') + if not os.path.exists(git_config_path): + return None + in_origin_remote = False + + with open(git_config_path, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('[remote "origin"]'): + in_origin_remote = True + elif in_origin_remote and line.startswith('url'): + url = line.split('=', maxsplit=1)[1].strip() + return url.split('/')[-1].split('.')[0] return None + + def _get_upstream_remote_name(self): + git_config_path = os.path.join(self._folder_path, '.git', 'config') + + if not self._repo_name: + return None + + if not os.path.exists(git_config_path): + return None + + remote_name = None + + with open(git_config_path, 'r') as f: + current_remote = None + for line in f: + line = line.strip() + if line.startswith('[remote "'): + current_remote = line[9:-2] # Extract remote name between quotes + elif line.startswith('url') and current_remote: + url = line.split('=', maxsplit=1)[1].strip() + # Check for both HTTPS and git@ formats + if (url == f"git@github.com:Azure/{self._repo_name}.git" or + url == f"https://github.com/Azure/{self._repo_name}.git"): + remote_name = current_remote + break + return remote_name + + def fetch_upstream(self, branch_name): + if not self._remote_name: + raise InvalidAPIUsage(f"Cannot find upstream for {self._repo_name} please run ` git -C {self._folder_path} remote add upstream https://github.com/Azure/{self._repo_name}.git`") + + try: + subprocess.run(args=["git", "-C", self._folder_path, "fetch", self._remote_name, branch_name], shell=False, check=True) + except Exception as e: + raise InvalidAPIUsage(f"Failed to fetch upstream for {self._repo_name} please run `git -C {self._folder_path} fetch {self._remote_name} {branch_name}`") + + def get_commit_hash_from_upstream(self, branch_name): + # First ensure we have the latest upstream branch + self.fetch_upstream(branch_name) + try: + # Get the merge-base commit (common ancestor) between HEAD and upstream branch + result = subprocess.run( + ["git", "-C", self._folder_path, "merge-base", "HEAD", f"{self._remote_name}/{branch_name}"], + capture_output=True, + text=True, + check=True + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + raise InvalidAPIUsage(f"Failed to get commit hash: {e.stderr}") class SingleModuleSwaggerSpecs: diff --git a/src/aaz_dev/utils/config.py b/src/aaz_dev/utils/config.py index 0eb7c731..3c319722 100644 --- a/src/aaz_dev/utils/config.py +++ b/src/aaz_dev/utils/config.py @@ -17,6 +17,7 @@ class Config: CLI_PATH = os.environ.get("AAZ_CLI_PATH", None) CLI_EXTENSION_PATH = os.environ.get("AAZ_CLI_EXTENSION_PATH", None) + POWERSHELL_PATH = os.environ.get("AAZ_POWERSHELL_PATH", None) AAZ_DEV_FOLDER = os.path.expanduser( os.environ.get("AAZ_DEV_FOLDER", os.path.join("~", ".aaz_dev")) @@ -106,6 +107,18 @@ def validate_and_setup_cli_extension_path(cls, ctx, param, value): raise ValueError(f"Path '{cls.CLI_EXTENSION_PATH}' does not exist.") return cls.CLI_EXTENSION_PATH + @classmethod + def validate_and_setup_powershell_path(cls, ctx, param, value): + # TODO: verify folder structure + if value != cls.POWERSHELL_PATH: + cls.POWERSHELL_PATH = os.path.expanduser(value) + if not os.path.exists(cls.POWERSHELL_PATH): + raise ValueError(f"Path '{cls.POWERSHELL_PATH}' does not exist.") + # verify the src/readme.azure.noprofile.md file exists + if not os.path.exists(os.path.join(cls.POWERSHELL_PATH, "src", "readme.azure.noprofile.md")): + raise ValueError(f"Path '{cls.POWERSHELL_PATH}' does not contain the required file 'src/readme.azure.noprofile.md', please make sure the code is based on the `generation` branch (https://github.com/Azure/azure-powershell/tree/generation).") + return cls.POWERSHELL_PATH + @classmethod def validate_and_setup_aaz_dev_workspace_folder(cls, ctx, param, value): # TODO: verify folder From ebe02d2489bbf87f987c317a76e7f3f9c81b8aec Mon Sep 17 00:00:00 2001 From: kai ru Date: Thu, 19 Dec 2024 15:04:17 +0800 Subject: [PATCH 13/18] implement to retrieve autorest configuration in modules --- src/aaz_dev/app/app.py | 3 +- src/aaz_dev/app/url_converters.py | 15 ++- src/aaz_dev/ps/api/powershell.py | 60 ++++++++++-- .../ps/controller/ps_module_manager.py | 93 +++++++++++++++++-- .../autorest/config_common_used_props.yaml | 67 +++++++++++++ src/aaz_dev/ps/tests/api_tests/__init__.py | 0 .../ps/tests/api_tests/test_powershell.py | 63 +++++++++++++ src/aaz_dev/ps/tests/common.py | 14 +++ 8 files changed, 295 insertions(+), 20 deletions(-) create mode 100644 src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml create mode 100644 src/aaz_dev/ps/tests/api_tests/__init__.py create mode 100644 src/aaz_dev/ps/tests/api_tests/test_powershell.py create mode 100644 src/aaz_dev/ps/tests/common.py diff --git a/src/aaz_dev/app/app.py b/src/aaz_dev/app/app.py index 90db0d45..dd1d3bce 100644 --- a/src/aaz_dev/app/app.py +++ b/src/aaz_dev/app/app.py @@ -27,12 +27,13 @@ def invalid_api_usage(e): return jsonify(e.to_dict()), e.status_code # register url converters - from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor + from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor, PSNamesPathConverter app.url_map.converters['base64'] = Base64Converter app.url_map.converters['name'] = NameConverter app.url_map.converters['Name'] = NameWithCapitalConverter app.url_map.converters['names_path'] = NamesPathConverter app.url_map.converters['list_path'] = ListPathConvertor + app.url_map.converters['PSNamesPath'] = PSNamesPathConverter # register routes of swagger module from swagger.api import register_blueprints diff --git a/src/aaz_dev/app/url_converters.py b/src/aaz_dev/app/url_converters.py index 1ca5b1f8..a6208039 100644 --- a/src/aaz_dev/app/url_converters.py +++ b/src/aaz_dev/app/url_converters.py @@ -32,7 +32,6 @@ def to_url(self, values): return super(NamesPathConverter, self).to_url(values) return '/'.join(super(NamesPathConverter, self).to_url(value) for value in values) - class ListPathConvertor(PathConverter): def to_python(self, value): @@ -43,5 +42,17 @@ def to_url(self, values): return super(ListPathConvertor, self).to_url(values) return '/'.join(super(ListPathConvertor, self).to_url(value) for value in values) +class PSNamesPathConverter(PathConverter): + regex = r"([A-Z][a-zA-Z0-9]*)/([A-Z][a-zA-Z0-9]*\.Autorest)" + weight = 200 + + def to_python(self, value): + return value.split('/') + + def to_url(self, values): + if isinstance(values, str): + return super(PSNamesPathConverter, self).to_url(values) + return '/'.join(super(PSNamesPathConverter, self).to_url(value) for value in values) + -__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor"] +__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor", "PSNamesPathConverter"] diff --git a/src/aaz_dev/ps/api/powershell.py b/src/aaz_dev/ps/api/powershell.py index 4965699c..3ba48c0e 100644 --- a/src/aaz_dev/ps/api/powershell.py +++ b/src/aaz_dev/ps/api/powershell.py @@ -2,8 +2,11 @@ from utils.config import Config from utils import exceptions -from command.controller.specs_manager import AAZSpecsManager +from ps.controller.ps_module_manager import PSModuleManager +from app.url_converters import PSNamesPathConverter +# from command.controller.specs_manager import AAZSpecsManager import logging +import re logging.basicConfig(level="INFO") @@ -11,14 +14,53 @@ bp = Blueprint('powershell', __name__, url_prefix='/PS/Powershell') -@bp.route("/Path", methods=("GET", "PUT")) +@bp.route("/Path", methods=("GET", )) def powershell_path(): + if Config.POWERSHELL_PATH is None: + raise exceptions.InvalidAPIUsage("PowerShell path is not set, please add `--ps` option to `aaz-dev run` command or set up `AAZ_POWERSHELL_PATH` environment variable") + return jsonify({"path": Config.POWERSHELL_PATH}) + + +@bp.route("/Modules", methods=("GET", "POST")) +def powershell_modules(): + manager = PSModuleManager() + if request.method == "GET": + modules = manager.list_modules() + result = [] + for module in modules: + result.append({ + **module, + 'url': url_for('powershell.powershell_module', module_names=module['name']), + }) + return jsonify(result) + elif request.method == "POST": + # create a new module in powershell + data = request.get_json() + if not data or not isinstance(data, dict) or 'name' not in data: + raise exceptions.InvalidAPIUsage("Invalid request body") + if not re.match(PSNamesPathConverter.regex, data['name'].split('/')): + raise exceptions.InvalidAPIUsage("Invalid module name") + module_names = data['name'].split('/') + # make sure the name is follow the PSNamesPathConverter.regex + module = manager.create_new_mod(module_names) + result = module.to_primitive() + result['url'] = url_for('powershell.powershell_module', module_names=module.name) + else: + raise NotImplementedError() + return jsonify(result) + + +@bp.route("/Modules/", methods=("GET", "PUT", "PATCH")) +def powershell_module(module_names): + manager = PSModuleManager() if request.method == "GET": - return jsonify({"path": Config.POWERSHELL_PATH}) + result = manager.load_module(module_names) + # result = module.to_primitive() + result['url'] = url_for('powershell.powershell_module', module_names=result['name']) elif request.method == "PUT": - data = request.json - try: - Config.validate_and_setup_powershell_path(None, None, data["path"]) - except ValueError as e: - raise exceptions.InvalidAPIUsage(str(e)) - return jsonify({"path": Config.POWERSHELL_PATH}) + raise NotImplementedError() + elif request.method == "PATCH": + raise NotImplementedError() + else: + raise NotImplementedError() + return jsonify(result) diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py index 264b9e2b..ed3a788b 100644 --- a/src/aaz_dev/ps/controller/ps_module_manager.py +++ b/src/aaz_dev/ps/controller/ps_module_manager.py @@ -1,17 +1,94 @@ -import ast -import glob -import json import logging import os -import pkgutil -import re +import yaml -from utils import exceptions from utils.config import Config -from collections import deque logger = logging.getLogger('backend') class PSModuleManager: - pass + + def __init__(self): + module_folder = self._find_module_folder() + self.folder = module_folder + + def _find_module_folder(self): + powershell_folder = Config.POWERSHELL_PATH + if not os.path.exists(powershell_folder) or not os.path.isdir(powershell_folder): + raise ValueError(f"Invalid PowerShell folder: '{powershell_folder}'") + module_folder = os.path.join(powershell_folder, "src") + if not os.path.exists(module_folder): + raise ValueError(f"Invalid PowerShell folder: cannot find modules in: '{module_folder}'") + return module_folder + + def list_modules(self): + modules = [] + for folder_name in os.listdir(self.folder): + path = os.path.join(self.folder, folder_name) + if os.path.isdir(path): + for sub_folder in os.listdir(path): + if os.path.isdir(os.path.join(path, sub_folder)) and sub_folder.endswith(".Autorest"): + name = f"{folder_name}/{sub_folder}" + modules.append({ + "name": name, + "folder": os.path.join(path, sub_folder) + }) + return sorted(modules, key=lambda a: a['name']) + + def create_new_mod(self, module_names): + if isinstance(module_names, str): + module_names = module_names.split('/') + folder = os.path.join(self.folder, *module_names) + os.makedirs(folder, exist_ok=True) + + def load_module(self, module_names): + if isinstance(module_names, str): + module_names = module_names.split('/') + folder = os.path.join(self.folder, *module_names) + if not os.path.exists(folder): + raise ValueError(f"Module folder not found: '{folder}'") + autorest_config = self.load_autorest_config(module_names) + return { + **autorest_config, + "name": "/".join(module_names), + "folder": folder + } + + def load_autorest_config(self, module_names): + if isinstance(module_names, str): + module_names = module_names.split('/') + folder = os.path.join(self.folder, *module_names) + readme_file = os.path.join(folder, "README.md") + if not os.path.exists(readme_file): + raise ValueError(f"README.md not found in: '{readme_file}'") + with open(readme_file, "r") as f: + content = f.readlines() + autorest_config = [] + in_autorest_config_section = False + in_yaml_section = False + for line in content: + if line.strip().startswith("### AutoRest Configuration"): + in_autorest_config_section = True + elif in_autorest_config_section: + if line.strip().startswith("###"): + break + if line.strip().startswith("```") and 'yaml' in line: + in_yaml_section = True + elif in_yaml_section: + if line.strip().startswith("```"): + in_yaml_section = False + else: + if line.strip(): + autorest_config.append(line) + else: + autorest_config.append("") + autorest_config_raw = "\n".join(autorest_config) + try: + yaml_config = yaml.load(autorest_config_raw, Loader=yaml.FullLoader) + except Exception as e: + raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_file}") + return { + "autorest_config": yaml_config, + # "raw": autorest_config_raw # can be used for directive merging + } diff --git a/src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml b/src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml new file mode 100644 index 00000000..7fb8a069 --- /dev/null +++ b/src/aaz_dev/ps/templates/autorest/config_common_used_props.yaml @@ -0,0 +1,67 @@ +metadata: # display to the users + dict: + authors: Microsoft Corporation + companyName: Microsoft Corporation + copyright: Microsoft Corporation. All rights reserved. + description: 'Microsoft Azure PowerShell: Portal Dashboard cmdlets' + licenseUri: https://aka.ms/azps-license + owners: Microsoft Corporation + projectUri: https://github.com/Azure/azure-powershell + releaseNotes: Initial release of Portal Dashboard cmdlets. + requireLicenseAcceptance: true + scriptsToProcess: + - ./custom/Helpers.ps1 + tags: Azure ResourceManager ARM PSModule Portal Dashboard +prefix: # the variable + basic: + - Az # the default value +service-name: # by default it's calculated + basic: + - ADDomainServices + - CostManagement +module-name: + basic: + - $(prefix).$(service-name) # the default value + - Az.SignalR +root-module-name: # used for sub module to generate the code in root module if there are multiple sub modules + basic: + - $(prefix).AlertsManagement + - $(prefix).AppConfiguration +namespace: # used for sub module to define the powershell class namespace + basic: + - Microsoft.Azure.PowerShell.Cmdlets.$(service-name) # the default value + - Microsoft.Azure.PowerShell.Cmdlets.Metric +subject-prefix: # the subject prefix is used in the commandlet name, which will be added after {action}-AZ{subprefix} + basic: + - '' # no subject prefix included + - $(service-name) # the default value + - $Informatica + - $elastic + - AD +repo: # the variable, used to find the required files, it's used for input-files + basic: + - https://github.com/Azure/azure-rest-api-specs/tree/$(commit) # the default value, and also support for other repos +require: # required readme.md files for autorest consume + list: + - $(this-folder)/../../readme.azure.noprofile.md # always needed + - $(repo)/specification/appcomplianceautomation/resource-manager/readme.md + - $(repo)/specification/azurefleet/resource-manager/readme.md + - $(this-folder)/../../helpers/KeyVault/readme.noprofile.md + - $(this-folder)/../../helpers/ManagedIdentity/readme.noprofile.md +endpoint-resource-id-key-name: # data-plane used, the resource id key name should be registered in azure powershell account before. It will control the client endpoint and auth. +inlining-threshold: # flatten threshold, 40, which used to control the max allowed flatten properties into outer layer +sanitize-names: # TODO: need code for detail how to check the sanitize rules for the argument names + basic: + - true # the default value +support-json-input: # default value is true +tag: # used to filter out the input apis, the tag should be in the swagger readme.md file + basic: + - package-2022-10-12-preview + - package-2023-01-01 +try-require: # using the directive configuration if it exists as prefix + list: + - $(repo)/specification/azurefleet/resource-manager/readme.powershell.md + - $(repo)/specification/containerregistry/resource-manager/readme.powershell.md +use-extension: + dict: + '@autorest/powershell': 3.x # always this value diff --git a/src/aaz_dev/ps/tests/api_tests/__init__.py b/src/aaz_dev/ps/tests/api_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/aaz_dev/ps/tests/api_tests/test_powershell.py b/src/aaz_dev/ps/tests/api_tests/test_powershell.py new file mode 100644 index 00000000..0d5efd90 --- /dev/null +++ b/src/aaz_dev/ps/tests/api_tests/test_powershell.py @@ -0,0 +1,63 @@ +from ps.tests.common import CommandTestCase +from utils.config import Config +from utils.base64 import b64encode_str +from utils.stage import AAZStageEnum +# from cli.controller.az_module_manager import AzMainManager, AzExtensionManager +import os +import shutil +import yaml + + +class APIPowerShellTest(CommandTestCase): + + def test_get_powershell_path(self): + with self.app.test_client() as c: + print(Config.POWERSHELL_PATH) + rv = c.get("/PS/Powershell/Path") + self.assertTrue(rv.status_code == 200) + data = rv.get_json() + self.assertTrue(data["path"] == Config.POWERSHELL_PATH) + + def test_list_powershell_modules(self): + config_dict = {} + with self.app.test_client() as c: + rv = c.get("/PS/Powershell/Modules") + self.assertTrue(rv.status_code == 200) + data = rv.get_json() + self.assertTrue(len(data) > 100) + self.assertTrue(all(module["name"].endswith(".Autorest") for module in data)) + for module in data: + request_url = module["url"] + rv = c.get(request_url) + self.assertTrue(rv.status_code == 200) + data = rv.get_json() + if data["autorest_config"] is None: + continue + for key, value in data["autorest_config"].items(): + if key in ["directive", "commit", "input-file", "title", "module-version"]: + continue + if key not in config_dict: + config_dict[key] = { + "list": set(), + "dict": {}, + "basic": set(), + } + if isinstance(value, list): + config_dict[key]["list"].update(value) + elif isinstance(value, dict): + config_dict[key]["dict"].update(value) + else: + config_dict[key]["basic"].add(value) + for key, value in config_dict.items(): + if not len(value["list"]): + del value["list"] + else: + value["list"] = sorted(list(value["list"])) + if not len(value["dict"]): + del value["dict"] + if not len(value["basic"]): + del value["basic"] + else: + value["basic"] = sorted(list(value["basic"])) + # with open("ps/templates/autorest/config_common_used_props.yaml", "w") as f: + # yaml.dump(config_dict, f) diff --git a/src/aaz_dev/ps/tests/common.py b/src/aaz_dev/ps/tests/common.py new file mode 100644 index 00000000..915a4bf3 --- /dev/null +++ b/src/aaz_dev/ps/tests/common.py @@ -0,0 +1,14 @@ +from app.tests.common import ApiTestCase +from utils.config import Config +# from command.tests.common import workspace_name +# from swagger.utils.tools import swagger_resource_path_to_resource_id +# from swagger.utils.source import SourceTypeEnum +# from utils.plane import PlaneEnum +# from utils.stage import AAZStageEnum +# from utils.client import CloudEnum + + +class CommandTestCase(ApiTestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) From 29ca6084cc45a780416124b3aa495e8473dbfa2b Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 10:48:38 +0800 Subject: [PATCH 14/18] implement to load the powershell modules --- src/aaz_dev/ps/api/powershell.py | 4 +- .../autorest_configuration_generator.py | 3 +- .../ps/controller/ps_module_manager.py | 153 +++++++++++++----- src/aaz_dev/ps/model/__init__.py | 1 + src/aaz_dev/ps/model/_module_config.py | 81 ++++++++++ .../ps/tests/api_tests/test_powershell.py | 43 +---- .../swagger/model/specs/_resource_provider.py | 23 --- .../swagger/model/specs/_swagger_specs.py | 129 ++++++++------- .../src/views/cli/CLIModGeneratorToolBar.tsx | 12 +- src/web/src/views/cli/CLIModuleGenerator.tsx | 106 +++++++++++- .../src/views/workspace/WSEditorToolBar.tsx | 6 +- src/web/vite.config.ts | 1 + 12 files changed, 389 insertions(+), 173 deletions(-) create mode 100644 src/aaz_dev/ps/model/_module_config.py diff --git a/src/aaz_dev/ps/api/powershell.py b/src/aaz_dev/ps/api/powershell.py index 3ba48c0e..d5805b63 100644 --- a/src/aaz_dev/ps/api/powershell.py +++ b/src/aaz_dev/ps/api/powershell.py @@ -54,8 +54,8 @@ def powershell_modules(): def powershell_module(module_names): manager = PSModuleManager() if request.method == "GET": - result = manager.load_module(module_names) - # result = module.to_primitive() + module = manager.load_module(module_names) + result = module.to_primitive() result['url'] = url_for('powershell.powershell_module', module_names=result['name']) elif request.method == "PUT": raise NotImplementedError() diff --git a/src/aaz_dev/ps/controller/autorest_configuration_generator.py b/src/aaz_dev/ps/controller/autorest_configuration_generator.py index 0d18f862..46c0fa17 100644 --- a/src/aaz_dev/ps/controller/autorest_configuration_generator.py +++ b/src/aaz_dev/ps/controller/autorest_configuration_generator.py @@ -106,7 +106,8 @@ def generate_config(self): if not swagger_resources: raise ResourceNotFind("Resources not find in Swagger") - readme_parts= rp._readme_path.split(os.sep) + # TODO: use the correct readme file + readme_parts= rp._readme_paths[0].split(os.sep) ps_cfg.readme_file = '/'.join(readme_parts[readme_parts.index("specification"):]) ps_cfg.version = "0.1.0" ps_cfg.module_name = mod_names.split("/")[0] diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py index ed3a788b..511f767a 100644 --- a/src/aaz_dev/ps/controller/ps_module_manager.py +++ b/src/aaz_dev/ps/controller/ps_module_manager.py @@ -1,9 +1,15 @@ import logging import os -import yaml from utils.config import Config - +from utils.plane import PlaneEnum +from utils.readme_helper import parse_readme_file +from ps.model import PSModuleConfig +from swagger.controller.specs_manager import SwaggerSpecsManager +from swagger.model.specs import SwaggerModule +from command.controller.specs_manager import AAZSpecsManager +from swagger.model.specs import OpenAPIResourceProvider +from swagger.utils.tools import resolve_path_to_uri logger = logging.getLogger('backend') @@ -12,6 +18,20 @@ class PSModuleManager: def __init__(self): module_folder = self._find_module_folder() self.folder = module_folder + self._aaz_specs = None + self._swagger_specs = None + + @property + def aaz_specs(self): + if not self._aaz_specs: + self._aaz_specs = AAZSpecsManager() + return self._aaz_specs + + @property + def swagger_specs(self): + if not self._swagger_specs: + self._swagger_specs = SwaggerSpecsManager() + return self._swagger_specs def _find_module_folder(self): powershell_folder = Config.POWERSHELL_PATH @@ -48,12 +68,8 @@ def load_module(self, module_names): folder = os.path.join(self.folder, *module_names) if not os.path.exists(folder): raise ValueError(f"Module folder not found: '{folder}'") - autorest_config = self.load_autorest_config(module_names) - return { - **autorest_config, - "name": "/".join(module_names), - "folder": folder - } + config = self.load_module_config(module_names) + return config def load_autorest_config(self, module_names): if isinstance(module_names, str): @@ -62,33 +78,96 @@ def load_autorest_config(self, module_names): readme_file = os.path.join(folder, "README.md") if not os.path.exists(readme_file): raise ValueError(f"README.md not found in: '{readme_file}'") - with open(readme_file, "r") as f: - content = f.readlines() - autorest_config = [] - in_autorest_config_section = False - in_yaml_section = False - for line in content: - if line.strip().startswith("### AutoRest Configuration"): - in_autorest_config_section = True - elif in_autorest_config_section: - if line.strip().startswith("###"): - break - if line.strip().startswith("```") and 'yaml' in line: - in_yaml_section = True - elif in_yaml_section: - if line.strip().startswith("```"): - in_yaml_section = False - else: - if line.strip(): - autorest_config.append(line) - else: - autorest_config.append("") - autorest_config_raw = "\n".join(autorest_config) + content = parse_readme_file(readme_file) + return content['config'], content['title'] + + def load_module_config(self, module_names): try: - yaml_config = yaml.load(autorest_config_raw, Loader=yaml.FullLoader) - except Exception as e: - raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_file}") - return { - "autorest_config": yaml_config, - # "raw": autorest_config_raw # can be used for directive merging - } + autorest_config, readme_title = self.load_autorest_config(module_names) + except: + logger.error(f"Failed to load autorest config for module: {module_names}, error: {e}") + raise + + config = PSModuleConfig() + config.name = "/".join(module_names) + config.folder = self.folder + if not autorest_config: + raise ValueError(f"autorest config not found in README.md for module: {config.name}") + + # config.swagger = autorest_config + repo = autorest_config.get('repo', "https://github.com/Azure/azure-rest-api-specs/blob/$(commit)") + if commit := autorest_config.get('commit'): + repo = repo.replace("$(commit)", commit) + if "$(commit)" in repo: + # make sure the repo is valid https link or valid folder path + raise ValueError(f"commit is not defined in autorest config for module: {config.name}") + config.repo = repo + + readme_file = None + for required_file in autorest_config['require']: + if required_file.startswith('$(repo)/') and required_file.endswith('/readme.md'): + readme_file = required_file.replace('$(repo)/', '') + break + + if not readme_file: + # search the readme.md in the swagger specs folder + for input_file in autorest_config.get('input-file', []): + if "/specification/" in input_file: + folder_names = input_file.split("/specification/")[1].split("/")[:-1] + path = os.path.join(self.swagger_specs.specs.spec_folder_path, *folder_names) + while path != self.swagger_specs.specs.spec_folder_path: + if os.path.exists(os.path.join(path, "readme.md")): + readme_file = os.path.join(path, "readme.md") + break + path = os.path.dirname(path) + if readme_file: + readme_file = resolve_path_to_uri(readme_file) + break + if not readme_file: + raise ValueError(f"swagger readme.md not defined in autorest config for module: {config.name}") + + # use the local swagger specs to find the resource provider even the repo is in remote + # we can always suppose the local swagger specs will always be newer than the used commit in submitted azure.powershell code + rp = None + readme_config = None + plane = PlaneEnum.Mgmt if "resource-manager" in readme_file else PlaneEnum._Data + for module in self.swagger_specs.get_modules(plane): + module_relative_path = resolve_path_to_uri(module.folder_path) + "/" + if readme_file.startswith(module_relative_path): + for resource_provider in module.get_resource_providers(): + if not isinstance(resource_provider, OpenAPIResourceProvider): + continue + readme_config = resource_provider.load_readme_config(readme_file) + if readme_config: + rp = resource_provider + break + if rp: + break + if not rp: + raise ValueError(f"Resource provider not found in autorest config for module: {config.name}") + config.rp = rp + config.swagger = str(rp) + + if tag := autorest_config.get('tag'): + config.tag = tag + if input_files := autorest_config.get('input-file'): + config.input_files = [] + for input_file in input_files: + if input_file.startswith('$(repo)/'): + input_file = input_file.replace('$(repo)/', '') + config.input_files.append(input_file) + if not config.input_files and not config.tag: + config.tag = readme_config.get('tag', None) + + if readme_title.startswith("Az."): + config.service_name = readme_title.split(".")[1] + if title := autorest_config.get('title'): + config.title = title + else: + # get title from swagger readme + config.title = readme_config.get('title', None) + + if not config.title: + raise ValueError(f"Title not found in autorest config or swagger readme for module: {config.name}") + + return config diff --git a/src/aaz_dev/ps/model/__init__.py b/src/aaz_dev/ps/model/__init__.py index e69de29b..91275f15 100644 --- a/src/aaz_dev/ps/model/__init__.py +++ b/src/aaz_dev/ps/model/__init__.py @@ -0,0 +1 @@ +from ._module_config import PSModuleConfig diff --git a/src/aaz_dev/ps/model/_module_config.py b/src/aaz_dev/ps/model/_module_config.py new file mode 100644 index 00000000..92fa74bf --- /dev/null +++ b/src/aaz_dev/ps/model/_module_config.py @@ -0,0 +1,81 @@ + + +from schematics.models import Model +from schematics.types import ModelType, DictType, StringType, ListType + + +class PSModuleConfig(Model): + name = StringType(required=True) + folder = StringType(required=True) + repo = StringType(required=True) # swagger repo path, https://github.com/Azure//tree/ or $(this-folder)/../../../ + swagger = StringType(required=True) # swagger resource provider, //ResourceProviders/ + + # use tag or input files to select the swagger apis + tag = StringType() # if the tag selected, the input_files will be ignored + input_files = ListType( + StringType(), + serialized_name='inputFiles', + deserialize_from='inputFiles', + ) # The input file should not contain $(repo) and can be directly appended to the repo + + title = StringType(required=True) # the required value for the autorest configuration + service_name = StringType( + required=True, + serialized_name='serviceName', + deserialize_from='serviceName', + ) # by default calculated from the title with this implementation https://github.com/Azure/autorest.powershell/blob/main/powershell/plugins/plugin-tweak-model.ts#L25-L33 + + # those default value defined in the noprofile.md configuration https://github.com/Azure/azure-powershell/blob/generation/src/readme.azure.noprofile.md + module_name = StringType( + required=True, + serialized_name='moduleName', + deserialize_from='moduleName', + default='$(prefix).$(service-name)' + ) # by default $(prefix).$(service-name) + namespace = StringType( + required=True, + default='Microsoft.Azure.PowerShell.Cmdlets.$(service-name)' + ) # used for sub module to define the powershell class namespace, by default Microsoft.Azure.PowerShell.Cmdlets.$(service-name) + subject_prefix = StringType( + required=True, + serialized_name='subjectPrefix', + deserialize_from='subjectPrefix', + default='$(service-name)' + ) # the default value $(service-name) + # root_module_name = StringType() # used for sub module to generate the code in root module if there are multiple sub modules + + prefix = StringType( + default='Az', + ) # Not allowed to change + + class Options: + serialize_when_none = False + + # swagger related properties + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rp = None + + @property + def repo_name(self): + return self.repo.split('/tree/', 1)[0].split('/')[-1] + + @property + def commit(self): + parts = self.repo.split('/tree/', 1) + if len(parts) == 2: + return parts[1].split('/')[0] + return None + + @property + def plane(self): + return self.swagger.split('/')[0] + + @property + def mod_names(self): + return self.swagger.split("/ResourceProviders/")[0].split('/')[1:] + + @property + def rp_name(self): + return self.swagger.split("/ResourceProviders/")[1].split('/')[0] diff --git a/src/aaz_dev/ps/tests/api_tests/test_powershell.py b/src/aaz_dev/ps/tests/api_tests/test_powershell.py index 0d5efd90..45ac186d 100644 --- a/src/aaz_dev/ps/tests/api_tests/test_powershell.py +++ b/src/aaz_dev/ps/tests/api_tests/test_powershell.py @@ -1,11 +1,5 @@ from ps.tests.common import CommandTestCase from utils.config import Config -from utils.base64 import b64encode_str -from utils.stage import AAZStageEnum -# from cli.controller.az_module_manager import AzMainManager, AzExtensionManager -import os -import shutil -import yaml class APIPowerShellTest(CommandTestCase): @@ -19,7 +13,6 @@ def test_get_powershell_path(self): self.assertTrue(data["path"] == Config.POWERSHELL_PATH) def test_list_powershell_modules(self): - config_dict = {} with self.app.test_client() as c: rv = c.get("/PS/Powershell/Modules") self.assertTrue(rv.status_code == 200) @@ -27,37 +20,13 @@ def test_list_powershell_modules(self): self.assertTrue(len(data) > 100) self.assertTrue(all(module["name"].endswith(".Autorest") for module in data)) for module in data: + if module["name"] in [ + "Communication/EmailServicedata.Autorest", + "ManagedServiceIdentity/ManagedServiceIdentity.Autorest", "VoiceServices/VoiceServices.Autorest", + "Resources/MSGraph.Autorest", "Migrate/Migrate.Autorest" + ]: + continue request_url = module["url"] rv = c.get(request_url) self.assertTrue(rv.status_code == 200) data = rv.get_json() - if data["autorest_config"] is None: - continue - for key, value in data["autorest_config"].items(): - if key in ["directive", "commit", "input-file", "title", "module-version"]: - continue - if key not in config_dict: - config_dict[key] = { - "list": set(), - "dict": {}, - "basic": set(), - } - if isinstance(value, list): - config_dict[key]["list"].update(value) - elif isinstance(value, dict): - config_dict[key]["dict"].update(value) - else: - config_dict[key]["basic"].add(value) - for key, value in config_dict.items(): - if not len(value["list"]): - del value["list"] - else: - value["list"] = sorted(list(value["list"])) - if not len(value["dict"]): - del value["dict"] - if not len(value["basic"]): - del value["basic"] - else: - value["basic"] = sorted(list(value["basic"])) - # with open("ps/templates/autorest/config_common_used_props.yaml", "w") as f: - # yaml.dump(config_dict, f) diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index d98ba907..ce925954 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -80,29 +80,6 @@ def load_readme_config(self, readme_file): return parse_readme_file(readme_path)['config'] return None - @property - def default_tag(self): - if self._readme_path is None: - return None - - with open(self._readme_path, 'r', encoding='utf-8') as f: - readme = f.read() - lines = readme.split('\n') - for i in range(len(lines)): - line = lines[i] - if line.startswith('### Basic Information'): - lines = lines[i+1:] - break - latest_tag = None - for i in range(len(lines)): - line = lines[i] - if line.startswith('##'): - break - if line.startswith('tag:'): - latest_tag = line.split(':')[-1].strip() - break - return latest_tag - @property def tags(self): if self._tags is None: diff --git a/src/aaz_dev/swagger/model/specs/_swagger_specs.py b/src/aaz_dev/swagger/model/specs/_swagger_specs.py index 6b6e8dd4..a3e43252 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_specs.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_specs.py @@ -1,5 +1,4 @@ import os -import subprocess from utils.plane import PlaneEnum from utils.exceptions import ResourceNotFind, InvalidAPIUsage @@ -11,8 +10,8 @@ class SwaggerSpecs: def __init__(self, folder_path): self._folder_path = folder_path - self._repo_name = self._get_repo_name() - self._remote_name = self._get_upstream_remote_name() + # self._repo_name = self._get_repo_name() + # self._remote_name = self._get_upstream_remote_name() @property def spec_folder_path(self): @@ -72,73 +71,73 @@ def get_data_plane_module(self, *names, plane): return module return None - def _get_repo_name(self): - # get repo name from origin remote - git_config_path = os.path.join(self._folder_path, '.git', 'config') - if not os.path.exists(git_config_path): - return None - - in_origin_remote = False - - with open(git_config_path, 'r') as f: - for line in f: - line = line.strip() - if line.startswith('[remote "origin"]'): - in_origin_remote = True - elif in_origin_remote and line.startswith('url'): - url = line.split('=', maxsplit=1)[1].strip() - return url.split('/')[-1].split('.')[0] - return None + # def _get_repo_name(self): + # # get repo name from origin remote + # git_config_path = os.path.join(self._folder_path, '.git', 'config') + # if not os.path.exists(git_config_path): + # return None + + # in_origin_remote = False + + # with open(git_config_path, 'r') as f: + # for line in f: + # line = line.strip() + # if line.startswith('[remote "origin"]'): + # in_origin_remote = True + # elif in_origin_remote and line.startswith('url'): + # url = line.split('=', maxsplit=1)[1].strip() + # return url.split('/')[-1].split('.')[0] + # return None - def _get_upstream_remote_name(self): - git_config_path = os.path.join(self._folder_path, '.git', 'config') + # def _get_upstream_remote_name(self): + # git_config_path = os.path.join(self._folder_path, '.git', 'config') - if not self._repo_name: - return None + # if not self._repo_name: + # return None - if not os.path.exists(git_config_path): - return None + # if not os.path.exists(git_config_path): + # return None - remote_name = None + # remote_name = None - with open(git_config_path, 'r') as f: - current_remote = None - for line in f: - line = line.strip() - if line.startswith('[remote "'): - current_remote = line[9:-2] # Extract remote name between quotes - elif line.startswith('url') and current_remote: - url = line.split('=', maxsplit=1)[1].strip() - # Check for both HTTPS and git@ formats - if (url == f"git@github.com:Azure/{self._repo_name}.git" or - url == f"https://github.com/Azure/{self._repo_name}.git"): - remote_name = current_remote - break - return remote_name - - def fetch_upstream(self, branch_name): - if not self._remote_name: - raise InvalidAPIUsage(f"Cannot find upstream for {self._repo_name} please run ` git -C {self._folder_path} remote add upstream https://github.com/Azure/{self._repo_name}.git`") - - try: - subprocess.run(args=["git", "-C", self._folder_path, "fetch", self._remote_name, branch_name], shell=False, check=True) - except Exception as e: - raise InvalidAPIUsage(f"Failed to fetch upstream for {self._repo_name} please run `git -C {self._folder_path} fetch {self._remote_name} {branch_name}`") - - def get_commit_hash_from_upstream(self, branch_name): - # First ensure we have the latest upstream branch - self.fetch_upstream(branch_name) - try: - # Get the merge-base commit (common ancestor) between HEAD and upstream branch - result = subprocess.run( - ["git", "-C", self._folder_path, "merge-base", "HEAD", f"{self._remote_name}/{branch_name}"], - capture_output=True, - text=True, - check=True - ) - return result.stdout.strip() - except subprocess.CalledProcessError as e: - raise InvalidAPIUsage(f"Failed to get commit hash: {e.stderr}") + # with open(git_config_path, 'r') as f: + # current_remote = None + # for line in f: + # line = line.strip() + # if line.startswith('[remote "'): + # current_remote = line[9:-2] # Extract remote name between quotes + # elif line.startswith('url') and current_remote: + # url = line.split('=', maxsplit=1)[1].strip() + # # Check for both HTTPS and git@ formats + # if (url == f"git@github.com:Azure/{self._repo_name}.git" or + # url == f"https://github.com/Azure/{self._repo_name}.git"): + # remote_name = current_remote + # break + # return remote_name + + # def fetch_upstream(self, branch_name): + # if not self._remote_name: + # raise InvalidAPIUsage(f"Cannot find upstream for {self._repo_name} please run ` git -C {self._folder_path} remote add upstream https://github.com/Azure/{self._repo_name}.git`") + + # try: + # subprocess.run(args=["git", "-C", self._folder_path, "fetch", self._remote_name, branch_name], shell=False, check=True) + # except Exception as e: + # raise InvalidAPIUsage(f"Failed to fetch upstream for {self._repo_name} please run `git -C {self._folder_path} fetch {self._remote_name} {branch_name}`") + + # def get_commit_hash_from_upstream(self, branch_name): + # # First ensure we have the latest upstream branch + # self.fetch_upstream(branch_name) + # try: + # # Get the merge-base commit (common ancestor) between HEAD and upstream branch + # result = subprocess.run( + # ["git", "-C", self._folder_path, "merge-base", "HEAD", f"{self._remote_name}/{branch_name}"], + # capture_output=True, + # text=True, + # check=True + # ) + # return result.stdout.strip() + # except subprocess.CalledProcessError as e: + # raise InvalidAPIUsage(f"Failed to get commit hash: {e.stderr}") class SingleModuleSwaggerSpecs: diff --git a/src/web/src/views/cli/CLIModGeneratorToolBar.tsx b/src/web/src/views/cli/CLIModGeneratorToolBar.tsx index 99c78dff..e9c1ac37 100644 --- a/src/web/src/views/cli/CLIModGeneratorToolBar.tsx +++ b/src/web/src/views/cli/CLIModGeneratorToolBar.tsx @@ -14,11 +14,12 @@ interface CLIModGeneratorToolBarProps { moduleName: string; onHomePage: () => void; onGenerate: () => void; + onDraftPowerShell: () => void; } class CLIModGeneratorToolBar extends React.Component { render() { - const { moduleName, onHomePage, onGenerate } = this.props; + const { moduleName, onHomePage, onGenerate, onDraftPowerShell } = this.props; return ( + + + + + - diff --git a/src/web/src/views/cli/CLIModuleGenerator.tsx b/src/web/src/views/cli/CLIModuleGenerator.tsx index c007bd55..6cbbd08f 100644 --- a/src/web/src/views/cli/CLIModuleGenerator.tsx +++ b/src/web/src/views/cli/CLIModuleGenerator.tsx @@ -12,6 +12,10 @@ import { LinearProgress, Toolbar, Alert, + FormControl, + TextField, + InputAdornment, + IconButton, } from "@mui/material"; import { useParams } from "react-router"; import axios from "axios"; @@ -19,6 +23,7 @@ import CLIModGeneratorToolBar from "./CLIModGeneratorToolBar"; import CLIModGeneratorProfileCommandTree, { ExportModViewProfile, InitializeCommandTreeByModView, ProfileCommandTree } from "./CLIModGeneratorProfileCommandTree"; import CLIModGeneratorProfileTabs from "./CLIModGeneratorProfileTabs"; import { CLIModView, CLIModViewProfiles } from "./CLIModuleCommon"; +import { FolderOpen } from "@mui/icons-material"; interface CLISpecsSimpleCommand { names: string[], @@ -146,6 +151,7 @@ const CLIModuleGenerator: React.FC = ({ params }) => { const [commandTrees, setCommandTrees] = React.useState({}); const [selectedProfile, setSelectedProfile] = React.useState(undefined); const [showGenerateDialog, setShowGenerateDialog] = React.useState(false); + const [showDraftPowerShellDialog, setShowDraftPowerShellDialog] = React.useState(false); const fetchCommands = useSpecsCommandTree(); @@ -199,10 +205,18 @@ const CLIModuleGenerator: React.FC = ({ params }) => { setShowGenerateDialog(true); }; + const handleDraftPowerShell = () => { + setShowDraftPowerShellDialog(true); + }; + const handleGenerationClose = () => { setShowGenerateDialog(false); }; + const handleDraftPowerShellClose = () => { + setShowDraftPowerShellDialog(false); + }; + const onProfileChange = React.useCallback((selectedProfile: string) => { setSelectedProfile(selectedProfile); }, []); @@ -221,6 +235,7 @@ const CLIModuleGenerator: React.FC = ({ params }) => { moduleName={params.moduleName} onHomePage={handleBackToHomepage} onGenerate={handleGenerate} + onDraftPowerShell={handleDraftPowerShell} /> = ({ params }) => { onClose={handleGenerationClose} /> )} + {!showGenerateDialog && showDraftPowerShellDialog && ( + + )} theme.zIndex.drawer + 1 }} open={loading} @@ -378,7 +399,6 @@ function GenerateDialog(props: { }); } - return ( Generate CLI commands to {props.moduleName} @@ -401,6 +421,90 @@ function GenerateDialog(props: { ); } +function DraftPowerShellDialog(props: { + open: boolean; + onClose: (generated: boolean) => void; +}) { + + const [loading, setLoading] = React.useState(false); + const [updating, setUpdating] = React.useState(false); + const [invalidText, setInvalidText] = React.useState( + undefined + ); + const [powershellPath, setPowershellPath] = React.useState(undefined); + + const handleClose = () => { + props.onClose(false); + }; + + const handleDraftPowerShell = () => { + setUpdating(true); + // axios + // .post(`/CLI/Az/${props.repoName}/Modules/${props.moduleName}/DraftPowerShell`) + // .then(() => { + // setUpdating(false); + // props.onClose(true); + // }) + } + + React.useEffect(() => { + if (props.open) { + setLoading(true); + // call /PS/PowerShell/Path api to get the path of the PowerShell script + axios.get(`/PS/Powershell/Path`).then(res => { + setPowershellPath(res.data.path || undefined); + setLoading(false); + }).catch(err => { + console.error(err); + setLoading(false); + }); + } else { + setLoading(false); + } + }, [props.open]); + + return + Draft PowerShell Generation from CLI + + {invalidText && {invalidText} } + {loading && } + {!loading && + setPowershellPath(e.target.value)} + InputProps={{ + endAdornment: ( + + { + const dirHandler = await (window as any).showDirectoryPicker(); + const path = await dirHandler.resolve(); + setPowershellPath(path); + }} + > + + + + ) + }} + /> + } + + + {updating && + + + + } + {!updating && + + + } + + +} + const CLIModuleGeneratorWrapper = (props: any) => { const params = useParams(); return diff --git a/src/web/src/views/workspace/WSEditorToolBar.tsx b/src/web/src/views/workspace/WSEditorToolBar.tsx index eca3287a..b2751dbf 100644 --- a/src/web/src/views/workspace/WSEditorToolBar.tsx +++ b/src/web/src/views/workspace/WSEditorToolBar.tsx @@ -79,11 +79,7 @@ class WSEditorToolBar extends React.Component { - diff --git a/src/web/vite.config.ts b/src/web/vite.config.ts index d20433c5..0acfc3cc 100644 --- a/src/web/vite.config.ts +++ b/src/web/vite.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ open: true, proxy: { '/CLI': 'http://127.0.0.1:5000', + '/PS': 'http://127.0.0.1:5000', '/AAZ': 'http://127.0.0.1:5000', '/Swagger': 'http://127.0.0.1:5000', '/assets/typespec': 'http://127.0.0.1:5000', From cbee6bb47bf7e26469354249d0c3026916fe1b66 Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 12:34:32 +0800 Subject: [PATCH 15/18] verify the existance of input files --- .../ps/controller/ps_module_manager.py | 38 ++++++++++++------- .../ps/tests/api_tests/test_powershell.py | 14 +++++-- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py index 511f767a..745c33f0 100644 --- a/src/aaz_dev/ps/controller/ps_module_manager.py +++ b/src/aaz_dev/ps/controller/ps_module_manager.py @@ -4,9 +4,9 @@ from utils.config import Config from utils.plane import PlaneEnum from utils.readme_helper import parse_readme_file +from utils.exceptions import InvalidAPIUsage, VerificationError from ps.model import PSModuleConfig from swagger.controller.specs_manager import SwaggerSpecsManager -from swagger.model.specs import SwaggerModule from command.controller.specs_manager import AAZSpecsManager from swagger.model.specs import OpenAPIResourceProvider from swagger.utils.tools import resolve_path_to_uri @@ -36,10 +36,10 @@ def swagger_specs(self): def _find_module_folder(self): powershell_folder = Config.POWERSHELL_PATH if not os.path.exists(powershell_folder) or not os.path.isdir(powershell_folder): - raise ValueError(f"Invalid PowerShell folder: '{powershell_folder}'") + raise VerificationError(f"Invalid PowerShell folder: '{powershell_folder}'") module_folder = os.path.join(powershell_folder, "src") if not os.path.exists(module_folder): - raise ValueError(f"Invalid PowerShell folder: cannot find modules in: '{module_folder}'") + raise VerificationError(f"Invalid PowerShell folder: cannot find modules in: '{module_folder}'") return module_folder def list_modules(self): @@ -67,7 +67,7 @@ def load_module(self, module_names): module_names = module_names.split('/') folder = os.path.join(self.folder, *module_names) if not os.path.exists(folder): - raise ValueError(f"Module folder not found: '{folder}'") + raise VerificationError(f"Module folder not found: '{folder}'") config = self.load_module_config(module_names) return config @@ -77,7 +77,7 @@ def load_autorest_config(self, module_names): folder = os.path.join(self.folder, *module_names) readme_file = os.path.join(folder, "README.md") if not os.path.exists(readme_file): - raise ValueError(f"README.md not found in: '{readme_file}'") + raise VerificationError(f"README.md not found in: '{readme_file}'") content = parse_readme_file(readme_file) return content['config'], content['title'] @@ -92,7 +92,7 @@ def load_module_config(self, module_names): config.name = "/".join(module_names) config.folder = self.folder if not autorest_config: - raise ValueError(f"autorest config not found in README.md for module: {config.name}") + raise VerificationError(f"autorest config not found in README.md for module: {config.name}") # config.swagger = autorest_config repo = autorest_config.get('repo', "https://github.com/Azure/azure-rest-api-specs/blob/$(commit)") @@ -100,7 +100,7 @@ def load_module_config(self, module_names): repo = repo.replace("$(commit)", commit) if "$(commit)" in repo: # make sure the repo is valid https link or valid folder path - raise ValueError(f"commit is not defined in autorest config for module: {config.name}") + raise VerificationError(f"commit is not defined in autorest config for module: {config.name}") config.repo = repo readme_file = None @@ -124,7 +124,7 @@ def load_module_config(self, module_names): readme_file = resolve_path_to_uri(readme_file) break if not readme_file: - raise ValueError(f"swagger readme.md not defined in autorest config for module: {config.name}") + raise VerificationError(f"swagger readme.md not defined in autorest config for module: {config.name}") # use the local swagger specs to find the resource provider even the repo is in remote # we can always suppose the local swagger specs will always be newer than the used commit in submitted azure.powershell code @@ -144,7 +144,7 @@ def load_module_config(self, module_names): if rp: break if not rp: - raise ValueError(f"Resource provider not found in autorest config for module: {config.name}") + raise VerificationError(f"Resource provider not found in autorest config for module: {config.name}") config.rp = rp config.swagger = str(rp) @@ -153,11 +153,22 @@ def load_module_config(self, module_names): if input_files := autorest_config.get('input-file'): config.input_files = [] for input_file in input_files: - if input_file.startswith('$(repo)/'): - input_file = input_file.replace('$(repo)/', '') - config.input_files.append(input_file) + if '/specification/' in input_file: + file_path = input_file.split('/specification/')[1] + file_path = os.path.join(self.swagger_specs.specs.spec_folder_path, *file_path.split('/')) + if not os.path.exists(file_path): + raise VerificationError(f"Input file not found for module: {config.name}, input file: {file_path}") + config.input_files.append(file_path) if not config.input_files and not config.tag: + # using the tag from swagger readme config config.tag = readme_config.get('tag', None) + if config.tag and config.tag not in rp.tags: + raise VerificationError(f"Tag not found in resource provider for module: {config.name} with tag: {config.tag}") + if not config.input_files and config.tag: + config.input_files = list(rp.tags[config.tag]) + + if not config.input_files: + raise VerificationError(f"Input file not found in autorest config for module: {config.name}") if readme_title.startswith("Az."): config.service_name = readme_title.split(".")[1] @@ -168,6 +179,7 @@ def load_module_config(self, module_names): config.title = readme_config.get('title', None) if not config.title: - raise ValueError(f"Title not found in autorest config or swagger readme for module: {config.name}") + # TODO: get title from swagger json file + raise VerificationError(f"Title not found in autorest config or swagger readme for module: {config.name}") return config diff --git a/src/aaz_dev/ps/tests/api_tests/test_powershell.py b/src/aaz_dev/ps/tests/api_tests/test_powershell.py index 45ac186d..020e8760 100644 --- a/src/aaz_dev/ps/tests/api_tests/test_powershell.py +++ b/src/aaz_dev/ps/tests/api_tests/test_powershell.py @@ -19,13 +19,21 @@ def test_list_powershell_modules(self): data = rv.get_json() self.assertTrue(len(data) > 100) self.assertTrue(all(module["name"].endswith(".Autorest") for module in data)) + # start = None for module in data: if module["name"] in [ - "Communication/EmailServicedata.Autorest", - "ManagedServiceIdentity/ManagedServiceIdentity.Autorest", "VoiceServices/VoiceServices.Autorest", - "Resources/MSGraph.Autorest", "Migrate/Migrate.Autorest" + "Communication/EmailServicedata.Autorest", # cannot figure out the resource provider name for the data plane in this module + "ManagedServiceIdentity/ManagedServiceIdentity.Autorest", # invalid input files with duplicated paths in different versions + "MySql/MySql.Autorest", # swagger folder structure changed + "VoiceServices/VoiceServices.Autorest", # No title provided in the autorest config + "Resources/MSGraph.Autorest", # swagger not in the azure-rest-api-specs repo + "Migrate/Migrate.Autorest" # input files which contains multiple resource providers ]: continue + # if module["name"] == "MachineLearningServices/MachineLearningServices.Autorest": + # start = True + # if not start: + # continue request_url = module["url"] rv = c.get(request_url) self.assertTrue(rv.status_code == 200) From 378abac50e394e71ff321f679833755a7e75ca51 Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 14:56:54 +0800 Subject: [PATCH 16/18] support multiple resource providers in a powershell module --- .../ps/controller/ps_module_manager.py | 61 +++++++++++-------- src/aaz_dev/ps/model/_module_config.py | 22 +++---- .../ps/tests/api_tests/test_powershell.py | 6 -- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/aaz_dev/ps/controller/ps_module_manager.py b/src/aaz_dev/ps/controller/ps_module_manager.py index 745c33f0..56b8dd3b 100644 --- a/src/aaz_dev/ps/controller/ps_module_manager.py +++ b/src/aaz_dev/ps/controller/ps_module_manager.py @@ -82,6 +82,7 @@ def load_autorest_config(self, module_names): return content['config'], content['title'] def load_module_config(self, module_names): + # TODO: support multiple resource providers try: autorest_config, readme_title = self.load_autorest_config(module_names) except: @@ -103,16 +104,17 @@ def load_module_config(self, module_names): raise VerificationError(f"commit is not defined in autorest config for module: {config.name}") config.repo = repo - readme_file = None + readme_files = set() for required_file in autorest_config['require']: if required_file.startswith('$(repo)/') and required_file.endswith('/readme.md'): readme_file = required_file.replace('$(repo)/', '') - break + readme_files.add(readme_file) - if not readme_file: - # search the readme.md in the swagger specs folder + if not readme_files: + # search all the readme.md files from the input files for input_file in autorest_config.get('input-file', []): if "/specification/" in input_file: + readme_file = None folder_names = input_file.split("/specification/")[1].split("/")[:-1] path = os.path.join(self.swagger_specs.specs.spec_folder_path, *folder_names) while path != self.swagger_specs.specs.spec_folder_path: @@ -122,31 +124,34 @@ def load_module_config(self, module_names): path = os.path.dirname(path) if readme_file: readme_file = resolve_path_to_uri(readme_file) - break - if not readme_file: + readme_files.add(readme_file) + if not readme_files: raise VerificationError(f"swagger readme.md not defined in autorest config for module: {config.name}") # use the local swagger specs to find the resource provider even the repo is in remote # we can always suppose the local swagger specs will always be newer than the used commit in submitted azure.powershell code - rp = None - readme_config = None - plane = PlaneEnum.Mgmt if "resource-manager" in readme_file else PlaneEnum._Data - for module in self.swagger_specs.get_modules(plane): - module_relative_path = resolve_path_to_uri(module.folder_path) + "/" - if readme_file.startswith(module_relative_path): - for resource_provider in module.get_resource_providers(): - if not isinstance(resource_provider, OpenAPIResourceProvider): - continue - readme_config = resource_provider.load_readme_config(readme_file) - if readme_config: - rp = resource_provider + rps = [] + for readme_file in readme_files: + rp = None + readme_config = None + plane = PlaneEnum.Mgmt if "resource-manager" in readme_file else PlaneEnum._Data + for module in self.swagger_specs.get_modules(plane): + module_relative_path = resolve_path_to_uri(module.folder_path) + "/" + if readme_file.startswith(module_relative_path): + for resource_provider in module.get_resource_providers(): + if not isinstance(resource_provider, OpenAPIResourceProvider): + continue + readme_config = resource_provider.load_readme_config(readme_file) + if readme_config: + rp = resource_provider + break + if rp: break - if rp: - break - if not rp: - raise VerificationError(f"Resource provider not found in autorest config for module: {config.name}") - config.rp = rp - config.swagger = str(rp) + if not rp: + raise VerificationError(f"Resource provider not found in autorest config for module: {config.name} with readme file: {readme_file}") + rps.append(rp) + config.rps = rps + # config.swagger = str(rp) if tag := autorest_config.get('tag'): config.tag = tag @@ -162,10 +167,12 @@ def load_module_config(self, module_names): if not config.input_files and not config.tag: # using the tag from swagger readme config config.tag = readme_config.get('tag', None) - if config.tag and config.tag not in rp.tags: + if config.tag and len(rps) > 1: + raise VerificationError(f"Multiple resource providers found for module: {config.name} with tag: {config.tag}") + if config.tag and config.tag not in rps[0].tags: raise VerificationError(f"Tag not found in resource provider for module: {config.name} with tag: {config.tag}") if not config.input_files and config.tag: - config.input_files = list(rp.tags[config.tag]) + config.input_files = list(rps[0].tags[config.tag]) if not config.input_files: raise VerificationError(f"Input file not found in autorest config for module: {config.name}") @@ -177,7 +184,7 @@ def load_module_config(self, module_names): else: # get title from swagger readme config.title = readme_config.get('title', None) - + if not config.title: # TODO: get title from swagger json file raise VerificationError(f"Title not found in autorest config or swagger readme for module: {config.name}") diff --git a/src/aaz_dev/ps/model/_module_config.py b/src/aaz_dev/ps/model/_module_config.py index 92fa74bf..29100b2f 100644 --- a/src/aaz_dev/ps/model/_module_config.py +++ b/src/aaz_dev/ps/model/_module_config.py @@ -8,7 +8,7 @@ class PSModuleConfig(Model): name = StringType(required=True) folder = StringType(required=True) repo = StringType(required=True) # swagger repo path, https://github.com/Azure//tree/ or $(this-folder)/../../../ - swagger = StringType(required=True) # swagger resource provider, //ResourceProviders/ + # swagger = StringType(required=True) # swagger resource provider, //ResourceProviders/ # use tag or input files to select the swagger apis tag = StringType() # if the tag selected, the input_files will be ignored @@ -55,7 +55,7 @@ class Options: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.rp = None + self.rps = [] @property def repo_name(self): @@ -68,14 +68,14 @@ def commit(self): return parts[1].split('/')[0] return None - @property - def plane(self): - return self.swagger.split('/')[0] + # @property + # def plane(self): + # return self.swagger.split('/')[0] - @property - def mod_names(self): - return self.swagger.split("/ResourceProviders/")[0].split('/')[1:] + # @property + # def mod_names(self): + # return self.swagger.split("/ResourceProviders/")[0].split('/')[1:] - @property - def rp_name(self): - return self.swagger.split("/ResourceProviders/")[1].split('/')[0] + # @property + # def rp_name(self): + # return self.swagger.split("/ResourceProviders/")[1].split('/')[0] diff --git a/src/aaz_dev/ps/tests/api_tests/test_powershell.py b/src/aaz_dev/ps/tests/api_tests/test_powershell.py index 020e8760..4b065ae3 100644 --- a/src/aaz_dev/ps/tests/api_tests/test_powershell.py +++ b/src/aaz_dev/ps/tests/api_tests/test_powershell.py @@ -19,7 +19,6 @@ def test_list_powershell_modules(self): data = rv.get_json() self.assertTrue(len(data) > 100) self.assertTrue(all(module["name"].endswith(".Autorest") for module in data)) - # start = None for module in data: if module["name"] in [ "Communication/EmailServicedata.Autorest", # cannot figure out the resource provider name for the data plane in this module @@ -27,13 +26,8 @@ def test_list_powershell_modules(self): "MySql/MySql.Autorest", # swagger folder structure changed "VoiceServices/VoiceServices.Autorest", # No title provided in the autorest config "Resources/MSGraph.Autorest", # swagger not in the azure-rest-api-specs repo - "Migrate/Migrate.Autorest" # input files which contains multiple resource providers ]: continue - # if module["name"] == "MachineLearningServices/MachineLearningServices.Autorest": - # start = True - # if not start: - # continue request_url = module["url"] rv = c.get(request_url) self.assertTrue(rv.status_code == 200) From 0e44b16cb12b49da4497022c1984ab22db3da582 Mon Sep 17 00:00:00 2001 From: kai ru Date: Thu, 2 Jan 2025 17:03:22 +0800 Subject: [PATCH 17/18] Implement PSSketchProfile build and First UI page --- src/aaz_dev/app/run.py | 9 + src/aaz_dev/app/tests/common.py | 13 +- src/aaz_dev/ps/api/__init__.py | 3 +- src/aaz_dev/ps/api/editor.py | 24 + src/aaz_dev/ps/api/powershell.py | 13 +- .../controller/ps_sketch_profile_builder.py | 103 ++ src/aaz_dev/ps/controller/sketch_manager.py | 70 ++ src/aaz_dev/ps/model/sketch/__init__.py | 3 + src/aaz_dev/ps/model/sketch/_profile.py | 16 + src/aaz_dev/ps/model/sketch/_resource.py | 19 + .../ps/model/sketch/_resource_provider.py | 24 + .../ps/tests/api_tests/test_ps_editor.py | 19 + src/aaz_dev/ps/tests/common.py | 11 +- src/aaz_dev/utils/config.py | 12 + src/web/.gitignore | 3 + src/web/src/views/cli/CLIModuleGenerator.tsx | 1079 ++++++++++------- 16 files changed, 979 insertions(+), 442 deletions(-) create mode 100644 src/aaz_dev/ps/api/editor.py create mode 100644 src/aaz_dev/ps/controller/ps_sketch_profile_builder.py create mode 100644 src/aaz_dev/ps/controller/sketch_manager.py create mode 100644 src/aaz_dev/ps/model/sketch/__init__.py create mode 100644 src/aaz_dev/ps/model/sketch/_profile.py create mode 100644 src/aaz_dev/ps/model/sketch/_resource.py create mode 100644 src/aaz_dev/ps/model/sketch/_resource_provider.py create mode 100644 src/aaz_dev/ps/tests/api_tests/test_ps_editor.py diff --git a/src/aaz_dev/app/run.py b/src/aaz_dev/app/run.py index 2bc02851..fef13beb 100644 --- a/src/aaz_dev/app/run.py +++ b/src/aaz_dev/app/run.py @@ -102,6 +102,15 @@ def is_port_in_use(host, port): expose_value=False, help="The folder to load and save workspaces." ) +@click.option( + "--sketch-path", '-k', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + default=Config.AAZ_DEV_SKETCH_FOLDER, + required=not Config.AAZ_DEV_SKETCH_FOLDER, + callback=Config.validate_and_setup_aaz_dev_sketch_folder, + expose_value=False, + help="The folder to load and save sketches for PowerShell generation." +) @click.option( "--reload/--no-reload", default=None, diff --git a/src/aaz_dev/app/tests/common.py b/src/aaz_dev/app/tests/common.py index 3bdd50c4..418393e3 100644 --- a/src/aaz_dev/app/tests/common.py +++ b/src/aaz_dev/app/tests/common.py @@ -11,8 +11,10 @@ class ApiTestCase(TestCase): def __init__(self, *args, **kwargs): self.cleanup_dev_folder() - Config.AAZ_PATH = self.AAZ_FOLDER - Config.AAZ_DEV_FOLDER = self.AAZ_DEV_FOLDER + if self.AAZ_FOLDER is not None: + Config.AAZ_PATH = self.AAZ_FOLDER + if self.AAZ_DEV_FOLDER is not None: + Config.AAZ_DEV_FOLDER = self.AAZ_DEV_FOLDER Config.AAZ_DEV_WORKSPACE_FOLDER = os.path.join(self.AAZ_DEV_FOLDER, 'workspaces') super().__init__(*args, **kwargs) self.app = create_app() @@ -22,9 +24,12 @@ def __init__(self, *args, **kwargs): def cleanup_dev_folder(self): if os.path.exists(self.AAZ_DEV_FOLDER): shutil.rmtree(self.AAZ_DEV_FOLDER) - if os.path.exists(self.AAZ_FOLDER): + if self.AAZ_FOLDER is not None and os.path.exists(self.AAZ_FOLDER): shutil.rmtree(self.AAZ_FOLDER) def setUp(self): - os.makedirs(self.AAZ_FOLDER, exist_ok=True) + if self.AAZ_FOLDER is not None: + os.makedirs(self.AAZ_FOLDER, exist_ok=True) + if self.AAZ_DEV_FOLDER is not None: + os.makedirs(self.AAZ_DEV_FOLDER, exist_ok=True) diff --git a/src/aaz_dev/ps/api/__init__.py b/src/aaz_dev/ps/api/__init__.py index b05934ac..53949b7d 100644 --- a/src/aaz_dev/ps/api/__init__.py +++ b/src/aaz_dev/ps/api/__init__.py @@ -1,6 +1,7 @@ def register_blueprints(app): - from . import _cmds, autorest, powershell + from . import _cmds, autorest, powershell, editor app.register_blueprint(_cmds.bp) app.register_blueprint(autorest.bp) app.register_blueprint(powershell.bp) + app.register_blueprint(editor.bp) diff --git a/src/aaz_dev/ps/api/editor.py b/src/aaz_dev/ps/api/editor.py new file mode 100644 index 00000000..ada64a2f --- /dev/null +++ b/src/aaz_dev/ps/api/editor.py @@ -0,0 +1,24 @@ +import os + +from flask import Blueprint, jsonify, request, url_for, redirect +from utils import exceptions +from utils.config import Config +from cli.model.view import CLIViewProfile +from ps.controller.ps_sketch_profile_builder import PSSketchProfileBuilder + + +bp = Blueprint('ps_editor', __name__, url_prefix='/PS/Editor') + +@bp.route("/GenerateSketcheProfile", methods=("POST", )) +def generate_sketch_profile(): + if request.method == "POST": + data = request.get_json() + if not data or 'cliProfile' not in data: + raise exceptions.InvalidAPIUsage("Invalid request body") + cli_profile = CLIViewProfile(raw_data=data['cliProfile']) + cli_profile.validate() + builder = PSSketchProfileBuilder() + profile = builder(cli_profile) + return jsonify(profile.to_native()) + else: + raise NotImplementedError(request.method) diff --git a/src/aaz_dev/ps/api/powershell.py b/src/aaz_dev/ps/api/powershell.py index d5805b63..a6fa220e 100644 --- a/src/aaz_dev/ps/api/powershell.py +++ b/src/aaz_dev/ps/api/powershell.py @@ -50,17 +50,20 @@ def powershell_modules(): return jsonify(result) -@bp.route("/Modules/", methods=("GET", "PUT", "PATCH")) +@bp.route("/Modules/", methods=("GET",)) def powershell_module(module_names): manager = PSModuleManager() if request.method == "GET": module = manager.load_module(module_names) result = module.to_primitive() result['url'] = url_for('powershell.powershell_module', module_names=result['name']) - elif request.method == "PUT": - raise NotImplementedError() - elif request.method == "PATCH": - raise NotImplementedError() else: raise NotImplementedError() return jsonify(result) + + +@bp.route("/Modules//Generate", methods=("POST", )) +def powershell_module_generate(module_names): + manager = PSModuleManager() + manager.generate_module(module_names) + return jsonify({"message": "Module generated successfully"}) diff --git a/src/aaz_dev/ps/controller/ps_sketch_profile_builder.py b/src/aaz_dev/ps/controller/ps_sketch_profile_builder.py new file mode 100644 index 00000000..d5a59683 --- /dev/null +++ b/src/aaz_dev/ps/controller/ps_sketch_profile_builder.py @@ -0,0 +1,103 @@ +import logging +import os + +from utils.exceptions import ResourceNotFind +from ps.model.sketch import PSSketchProfile, PSSketchResourceProvider, PSSketchResource +from swagger.controller.specs_manager import SwaggerSpecsManager +from command.controller.specs_manager import AAZSpecsManager +from cli.model.view import CLIViewProfile +logger = logging.getLogger('backend') + + +class PSSketchProfileBuilder: + + def __init__(self, aaz_specs_manager: AAZSpecsManager=None, swagger_specs_manager: SwaggerSpecsManager=None): + self._aaz_specs = aaz_specs_manager + self._swagger_specs = swagger_specs_manager + + @property + def aaz_specs(self): + if not self._aaz_specs: + self._aaz_specs = AAZSpecsManager() + return self._aaz_specs + + @property + def swagger_specs(self): + if not self._swagger_specs: + self._swagger_specs = SwaggerSpecsManager() + return self._swagger_specs + + def __call__(self, cli_profile: CLIViewProfile): + profile = PSSketchProfile() + profile.resource_providers = [] + for cli_command in self.iter_cli_commands(cli_profile): + names = cli_command.names + version_name = cli_command.version + aaz_cmd = self.aaz_specs.find_command(*names) + if not aaz_cmd: + raise ResourceNotFind( + "Command '{}' not exist in AAZ".format(" ".join(names)) + ) + version = None + for v in aaz_cmd.versions or []: + if v.name == version_name: + version = v + break + if not version: + raise ResourceNotFind( + "Version '{}' of command '{}' not exist in AAZ".format( + version_name, " ".join(names) + ) + ) + resource = v.resources[0] + cfg = self.aaz_specs.load_resource_cfg_reader( + resource.plane, resource.id, resource.version + ) + if not cfg: + raise ResourceNotFind( + "Resource Configuration '{}' not exist in AAZ".format(resource.id) + ) + for resource in cfg.resources: + rp_swagger, _ = resource.swagger.split('/Paths/') + rp = None + for existing_rp in profile.resource_providers: + if existing_rp.swagger == rp_swagger: + rp = existing_rp + break + if rp is None: + rp = PSSketchResourceProvider() + rp.swagger = rp_swagger + rp.resources = [] + profile.resource_providers.append(rp) + s_resource = None + for existing_s_resource in rp.resources: + if existing_s_resource.id == resource.id: + s_resource = existing_s_resource + break + if s_resource is None: + s_resource = PSSketchResource() + s_resource.id = resource.id + s_resource.path = resource.path + s_resource.cli_commands = [] + s_resource.subresources = [] + rp.resources.append(s_resource) + s_resource.cli_commands.append(cli_command.__class__(raw_data=cli_command.to_native())) + if resource.subresource: + s_resource.subresources.append(resource.subresource) + return profile + + @classmethod + def iter_cli_commands(cls, profile: CLIViewProfile): + for command_group in profile.command_groups.values(): + for cli_command in cls._iter_cli_commands(command_group): + yield cli_command + + @classmethod + def _iter_cli_commands(cls, view_command_group): + if view_command_group.commands: + for cli_command in view_command_group.commands.values(): + yield cli_command + if view_command_group.command_groups: + for command_group in view_command_group.command_groups.values(): + for cli_command in cls._iter_cli_commands(command_group): + yield cli_command \ No newline at end of file diff --git a/src/aaz_dev/ps/controller/sketch_manager.py b/src/aaz_dev/ps/controller/sketch_manager.py new file mode 100644 index 00000000..7ca4ea35 --- /dev/null +++ b/src/aaz_dev/ps/controller/sketch_manager.py @@ -0,0 +1,70 @@ +import json +import logging +import os +import shutil +from datetime import datetime + +from swagger.controller.specs_manager import SwaggerSpecsManager +from command.controller.specs_manager import AAZSpecsManager +from cli.model.view import CLIViewProfile +from utils.config import Config + +logger = logging.getLogger('backend') + +class SketchManager: + + IN_MEMORY = "__IN_MEMORY_SKETCH__" + + @classmethod + def list_sketches(cls): + sketches = [] + if not os.path.exists(Config.AAZ_DEV_SKETCH_FOLDER): + return sketches + + for name in os.listdir(Config.AAZ_DEV_SKETCH_FOLDER): + if not os.path.isdir(os.path.join(Config.AAZ_DEV_SKETCH_FOLDER, name)): + continue + manager = cls(name) + if os.path.exists(manager.path) and os.path.isfile(manager.path): + sketches.append({ + "name": name, + "folder": os.path.join(Config.AAZ_DEV_SKETCH_FOLDER, name), + "updated": os.path.getmtime(os.path.join(Config.AAZ_DEV_SKETCH_FOLDER, name)) + }) + return sketches + + @classmethod + def new(cls, name, ps_module_name, selected_resource_providers, cli_profile: CLIViewProfile, **kwargs): + pass + + def __init__(self, name, folder=None): + self.name = name + if not folder: + if not Config.AAZ_DEV_SKETCH_FOLDER or os.path.exists(Config.AAZ_DEV_SKETCH_FOLDER) and not os.path.isdir(Config.AAZ_DEV_SKETCH_FOLDER): + raise ValueError(f"Invalid AAZ_DEV_SKETCH_FOLDER: Expect a folder path: {Config.AAZ_DEV_SKETCH_FOLDER}") + self.folder = os.path.join(Config.AAZ_DEV_SKETCH_FOLDER, name) + else: + self.folder = os.path.expanduser(folder) if folder != self.IN_MEMORY else self.IN_MEMORY + + if not self.is_in_memory and os.path.exists(self.folder) and not os.path.isdir(self.folder): + raise ValueError(f"Invalid sketch folder: Expect a folder path: {self.folder}") + self.path = os.path.join(self.folder, 'sketch.json') + + self.sketch = None + self._cfg_editors = {} + + @property + def is_in_memory(self): + return self.folder == self.IN_MEMORY + + @property + def aaz_specs(self): + if not self._aaz_specs: + self._aaz_specs = AAZSpecsManager() + return self._aaz_specs + + @property + def swagger_specs(self): + if not self._swagger_specs: + self._swagger_specs = SwaggerSpecsManager() + return self._swagger_specs diff --git a/src/aaz_dev/ps/model/sketch/__init__.py b/src/aaz_dev/ps/model/sketch/__init__.py new file mode 100644 index 00000000..8522ca09 --- /dev/null +++ b/src/aaz_dev/ps/model/sketch/__init__.py @@ -0,0 +1,3 @@ +from ._profile import PSSketchProfile +from ._resource_provider import PSSketchResourceProvider +from ._resource import PSSketchResource diff --git a/src/aaz_dev/ps/model/sketch/_profile.py b/src/aaz_dev/ps/model/sketch/_profile.py new file mode 100644 index 00000000..862354fe --- /dev/null +++ b/src/aaz_dev/ps/model/sketch/_profile.py @@ -0,0 +1,16 @@ +from schematics.models import Model +from schematics.types import StringType, ListType, ModelType +from ._resource_provider import PSSketchResourceProvider + +class PSSketchProfile(Model): + + resource_providers = ListType( + field=ModelType(PSSketchResourceProvider), + serialized_name="resourceProviders", + deserialize_from="resourceProviders", + ) + + class Options: + serialize_when_none = False + + diff --git a/src/aaz_dev/ps/model/sketch/_resource.py b/src/aaz_dev/ps/model/sketch/_resource.py new file mode 100644 index 00000000..453311f8 --- /dev/null +++ b/src/aaz_dev/ps/model/sketch/_resource.py @@ -0,0 +1,19 @@ +from schematics.models import Model +from schematics.types import StringType, ListType, ModelType +from command.model.configuration._fields import CMDResourceIdField, CMDVersionField +from cli.model.view import CLIViewCommand + + +class PSSketchResource(Model): + + id = CMDResourceIdField(required=True) + path = StringType(required=True) + cli_commands = ListType( + ModelType(CLIViewCommand), + serialized_name="cliCommands", + deserialize_from="cliCommands", + ) + subresources = ListType(StringType) + + class Options: + serialize_when_none = False diff --git a/src/aaz_dev/ps/model/sketch/_resource_provider.py b/src/aaz_dev/ps/model/sketch/_resource_provider.py new file mode 100644 index 00000000..a193cb93 --- /dev/null +++ b/src/aaz_dev/ps/model/sketch/_resource_provider.py @@ -0,0 +1,24 @@ +from schematics.models import Model +from schematics.types import StringType, ListType, ModelType +from ._resource import PSSketchResource + + +class PSSketchResourceProvider(Model): + + resources = ListType(ModelType(PSSketchResource)) + swagger = StringType(required=True) # swagger resource provider, //ResourceProviders/ + + class Options: + serialize_when_none = False + + @property + def plane(self): + return self.swagger.split('/')[0] + + @property + def mod_names(self): + return self.swagger.split("/ResourceProviders/")[0].split('/')[1:] + + @property + def rp_name(self): + return self.swagger.split("/ResourceProviders/")[1].split('/')[0] diff --git a/src/aaz_dev/ps/tests/api_tests/test_ps_editor.py b/src/aaz_dev/ps/tests/api_tests/test_ps_editor.py new file mode 100644 index 00000000..7830fd9f --- /dev/null +++ b/src/aaz_dev/ps/tests/api_tests/test_ps_editor.py @@ -0,0 +1,19 @@ +from ps.tests.common import CommandTestCase +from utils.config import Config + + +class APIPowerShellTest(CommandTestCase): + + def test_generate_sketch_profile(self): + mod_name = "monitor" + with self.app.test_client() as c: + rv = c.get(f"/CLI/Az/Main/Modules/{mod_name}") + self.assertTrue(rv.status_code == 200) + data = rv.get_json() + + latest_profile = data["profiles"]["latest"] + + rv = c.post("/PS/Editor/GenerateSketcheProfile", json={"cliProfile": latest_profile}) + self.assertTrue(rv.status_code == 200) + data = rv.get_json() + self.assertTrue(len(data["resourceProviders"]) > 1) diff --git a/src/aaz_dev/ps/tests/common.py b/src/aaz_dev/ps/tests/common.py index 915a4bf3..b26eaa92 100644 --- a/src/aaz_dev/ps/tests/common.py +++ b/src/aaz_dev/ps/tests/common.py @@ -1,14 +1,5 @@ from app.tests.common import ApiTestCase from utils.config import Config -# from command.tests.common import workspace_name -# from swagger.utils.tools import swagger_resource_path_to_resource_id -# from swagger.utils.source import SourceTypeEnum -# from utils.plane import PlaneEnum -# from utils.stage import AAZStageEnum -# from utils.client import CloudEnum - class CommandTestCase(ApiTestCase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + AAZ_FOLDER = None diff --git a/src/aaz_dev/utils/config.py b/src/aaz_dev/utils/config.py index 3c319722..f2c4fbe6 100644 --- a/src/aaz_dev/utils/config.py +++ b/src/aaz_dev/utils/config.py @@ -25,6 +25,9 @@ class Config: AAZ_DEV_WORKSPACE_FOLDER = os.path.expanduser( os.environ.get("AAZ_DEV_WORKSPACE_FOLDER", os.path.join(AAZ_DEV_FOLDER, "workspaces")) ) + AAZ_DEV_SKETCH_FOLDER = os.path.expanduser( + os.environ.get("AAZ_DEV_SKETCH_FOLDER", os.path.join(AAZ_DEV_FOLDER, "sketches")) + ) # Flask configurations HOST = os.environ.get("AAZ_HOST", '127.0.0.1') @@ -128,6 +131,15 @@ def validate_and_setup_aaz_dev_workspace_folder(cls, ctx, param, value): raise ValueError(f"Path '{cls.AAZ_DEV_WORKSPACE_FOLDER}' is not a folder.") return cls.AAZ_DEV_WORKSPACE_FOLDER + @classmethod + def validate_and_setup_aaz_dev_sketch_folder(cls, ctx, param, value): + # TODO: verify folder + if value != cls.AAZ_DEV_SKETCH_FOLDER: + cls.AAZ_DEV_SKETCH_FOLDER = os.path.expanduser(value) + if os.path.exists(cls.AAZ_DEV_SKETCH_FOLDER) and not os.path.isdir(cls.AAZ_DEV_SKETCH_FOLDER): + raise ValueError(f"Path '{cls.AAZ_DEV_SKETCH_FOLDER}' is not a folder.") + return cls.AAZ_DEV_SKETCH_FOLDER + @classmethod def get_swagger_root(cls): if cls.SWAGGER_PATH: diff --git a/src/web/.gitignore b/src/web/.gitignore index 4d29575d..bde3ef65 100644 --- a/src/web/.gitignore +++ b/src/web/.gitignore @@ -21,3 +21,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log* + +/.vscode + diff --git a/src/web/src/views/cli/CLIModuleGenerator.tsx b/src/web/src/views/cli/CLIModuleGenerator.tsx index 6cbbd08f..ca0b8a1c 100644 --- a/src/web/src/views/cli/CLIModuleGenerator.tsx +++ b/src/web/src/views/cli/CLIModuleGenerator.tsx @@ -1,514 +1,749 @@ import * as React from "react"; import { - Backdrop, - Box, - Button, - CircularProgress, - Dialog, - DialogActions, - DialogContent, - DialogTitle, - Drawer, - LinearProgress, - Toolbar, - Alert, - FormControl, - TextField, - InputAdornment, - IconButton, + Backdrop, + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + Drawer, + LinearProgress, + Toolbar, + Alert, + FormControl, + TextField, + Autocomplete, + createFilterOptions, + Checkbox, + FormControlLabel, + FormLabel, + FormGroup, } from "@mui/material"; import { useParams } from "react-router"; import axios from "axios"; import CLIModGeneratorToolBar from "./CLIModGeneratorToolBar"; -import CLIModGeneratorProfileCommandTree, { ExportModViewProfile, InitializeCommandTreeByModView, ProfileCommandTree } from "./CLIModGeneratorProfileCommandTree"; +import CLIModGeneratorProfileCommandTree, { + ExportModViewProfile, + InitializeCommandTreeByModView, + ProfileCommandTree, +} from "./CLIModGeneratorProfileCommandTree"; import CLIModGeneratorProfileTabs from "./CLIModGeneratorProfileTabs"; import { CLIModView, CLIModViewProfiles } from "./CLIModuleCommon"; -import { FolderOpen } from "@mui/icons-material"; interface CLISpecsSimpleCommand { - names: string[], + names: string[]; } interface CLISpecsSimpleCommands { - [name: string]: CLISpecsSimpleCommand, + [name: string]: CLISpecsSimpleCommand; } interface CLISpecsSimpleCommandGroups { - [name: string]: CLISpecsSimpleCommandGroup, + [name: string]: CLISpecsSimpleCommandGroup; } interface CLISpecsSimpleCommandGroup { - names: string[], - commands: CLISpecsSimpleCommands, - commandGroups: CLISpecsSimpleCommandGroups, + names: string[]; + commands: CLISpecsSimpleCommands; + commandGroups: CLISpecsSimpleCommandGroups; } interface CLISpecsSimpleCommandTree { - root: CLISpecsSimpleCommandGroup, + root: CLISpecsSimpleCommandGroup; } interface CLISpecsHelp { - short: string, - lines?: string[], + short: string; + lines?: string[]; } interface CLISpecsResource { - plane: string, - id: string, - version: string, - subresource?: string, + plane: string; + id: string; + version: string; + subresource?: string; } interface CLISpecsCommandExample { - name: string, - commands: string[], + name: string; + commands: string[]; } interface CLISpecsCommandVersion { - name: string, - stage?: string, - resources: CLISpecsResource[], - examples?: CLISpecsCommandExample[], + name: string; + stage?: string; + resources: CLISpecsResource[]; + examples?: CLISpecsCommandExample[]; } interface CLISpecsCommand { - names: string[], - help: CLISpecsHelp, - versions: CLISpecsCommandVersion[], + names: string[]; + help: CLISpecsHelp; + versions: CLISpecsCommandVersion[]; } interface CLISpecsCommandGroup { - names: string[], - help?: CLISpecsHelp, - commands?: CLISpecsCommands, - commandGroups?: CLISpecsCommandGroups, + names: string[]; + help?: CLISpecsHelp; + commands?: CLISpecsCommands; + commandGroups?: CLISpecsCommandGroups; } interface CLISpecsCommandGroups { - [name: string]: CLISpecsCommandGroup, + [name: string]: CLISpecsCommandGroup; } interface CLISpecsCommands { - [name: string]: CLISpecsCommand, + [name: string]: CLISpecsCommand; } async function retrieveCommand(names: string[]): Promise { - return axios.get(`/AAZ/Specs/CommandTree/Nodes/aaz/${names.slice(0, -1).join('/')}/Leaves/${names[names.length - 1]}`).then(res => res.data); + return axios + .get( + `/AAZ/Specs/CommandTree/Nodes/aaz/${names.slice(0, -1).join("/")}/Leaves/${names[names.length - 1]}` + ) + .then((res) => res.data); } -async function retrieveCommands(namesList: string[][]): Promise { - const namesListData = namesList.map(names => ['aaz', ...names]); - return axios.post(`/AAZ/Specs/CommandTree/Nodes/Leaves`, namesListData).then(res => res.data); +async function retrieveCommands( + namesList: string[][] +): Promise { + const namesListData = namesList.map((names) => ["aaz", ...names]); + return axios + .post(`/AAZ/Specs/CommandTree/Nodes/Leaves`, namesListData) + .then((res) => res.data); } -const useSpecsCommandTree: () => (namesList: string[][]) => Promise = () => { - const commandCache = React.useRef(new Map>()); - - const fetchCommands = React.useCallback(async (namesList: string[][]) => { - const promiseResults = []; - const uncachedNamesList = []; - for (const names of namesList) { - const cachedPromise = commandCache.current.get(names.join('/')); - if (!cachedPromise) { - uncachedNamesList.push(names); - } else { - promiseResults.push(cachedPromise); - } - } - if (uncachedNamesList.length === 0) { - return await Promise.all(promiseResults); - } else if (uncachedNamesList.length === 1) { - const commandPromise = retrieveCommand(uncachedNamesList[0]); - commandCache.current.set(uncachedNamesList[0].join('/'), commandPromise); - return (await Promise.all(promiseResults.concat(commandPromise))); +const useSpecsCommandTree: () => ( + namesList: string[][] +) => Promise = () => { + const commandCache = React.useRef( + new Map>() + ); + + const fetchCommands = React.useCallback( + async (namesList: string[][]) => { + const promiseResults = []; + const uncachedNamesList = []; + for (const names of namesList) { + const cachedPromise = commandCache.current.get(names.join("/")); + if (!cachedPromise) { + uncachedNamesList.push(names); } else { - const uncachedCommandsPromise = retrieveCommands(uncachedNamesList); - uncachedNamesList.forEach((names, idx) => { - commandCache.current.set(names.join('/'), uncachedCommandsPromise.then(commands => commands[idx])); - }); - return (await Promise.all(promiseResults)).concat(await uncachedCommandsPromise); + promiseResults.push(cachedPromise); } - }, [commandCache]); - return fetchCommands; -} - + } + if (uncachedNamesList.length === 0) { + return await Promise.all(promiseResults); + } else if (uncachedNamesList.length === 1) { + const commandPromise = retrieveCommand(uncachedNamesList[0]); + commandCache.current.set( + uncachedNamesList[0].join("/"), + commandPromise + ); + return await Promise.all(promiseResults.concat(commandPromise)); + } else { + const uncachedCommandsPromise = retrieveCommands(uncachedNamesList); + uncachedNamesList.forEach((names, idx) => { + commandCache.current.set( + names.join("/"), + uncachedCommandsPromise.then((commands) => commands[idx]) + ); + }); + return (await Promise.all(promiseResults)).concat( + await uncachedCommandsPromise + ); + } + }, + [commandCache] + ); + return fetchCommands; +}; interface ProfileCommandTrees { - [name: string]: ProfileCommandTree, + [name: string]: ProfileCommandTree; } interface CLIModuleGeneratorProps { - params: { - repoName: string; - moduleName: string; - }; + params: { + repoName: string; + moduleName: string; + }; } const CLIModuleGenerator: React.FC = ({ params }) => { - const [loading, setLoading] = React.useState(false); - const [invalidText, setInvalidText] = React.useState(undefined); - const [profiles, setProfiles] = React.useState([]); - const [commandTrees, setCommandTrees] = React.useState({}); - const [selectedProfile, setSelectedProfile] = React.useState(undefined); - const [showGenerateDialog, setShowGenerateDialog] = React.useState(false); - const [showDraftPowerShellDialog, setShowDraftPowerShellDialog] = React.useState(false); - - const fetchCommands = useSpecsCommandTree(); - - React.useEffect(() => { - loadModule(); - }, []); - - const loadModule = async () => { - try { - setLoading(true); - const profiles: string[] = await axios.get(`/CLI/Az/Profiles`).then(res => res.data); - - const modView: CLIModView = await axios.get(`/CLI/Az/${params.repoName}/Modules/${params.moduleName}`).then(res => res.data); - - const simpleTree: CLISpecsSimpleCommandTree = await axios.get(`/AAZ/Specs/CommandTree/Simple`).then(res => res.data); - - Object.keys(modView!.profiles).forEach((profile) => { - const idx = profiles.findIndex(v => v === profile); - if (idx === -1) { - throw new Error(`Invalid profile ${profile}`); - } - }); - - const commandTrees = Object.fromEntries(profiles.map((profile) => { - return [profile, InitializeCommandTreeByModView(profile, modView!.profiles[profile] ?? null, simpleTree)]; - })); - - const selectedProfile = profiles.length > 0 ? profiles[0] : undefined; - setProfiles(profiles); - setCommandTrees(commandTrees); - setSelectedProfile(selectedProfile); - setLoading(false); - } catch (err: any) { - console.error(err); - if (err.response?.data?.message) { - const data = err.response!.data!; - setInvalidText(`ResponseError: ${data.message!}`); - } else { - setInvalidText(`Error: ${err}`); - } + const [loading, setLoading] = React.useState(false); + const [invalidText, setInvalidText] = React.useState( + undefined + ); + const [profiles, setProfiles] = React.useState([]); + const [commandTrees, setCommandTrees] = React.useState( + {} + ); + const [selectedProfile, setSelectedProfile] = React.useState< + string | undefined + >(undefined); + const [showGenerateDialog, setShowGenerateDialog] = React.useState(false); + const [showDraftPowerShellDialog, setShowDraftPowerShellDialog] = + React.useState(false); + + const fetchCommands = useSpecsCommandTree(); + + React.useEffect(() => { + loadModule(); + }, []); + + const loadModule = async () => { + try { + setLoading(true); + const profiles: string[] = await axios + .get(`/CLI/Az/Profiles`) + .then((res) => res.data); + + const modView: CLIModView = await axios + .get(`/CLI/Az/${params.repoName}/Modules/${params.moduleName}`) + .then((res) => res.data); + + const simpleTree: CLISpecsSimpleCommandTree = await axios + .get(`/AAZ/Specs/CommandTree/Simple`) + .then((res) => res.data); + + Object.keys(modView!.profiles).forEach((profile) => { + const idx = profiles.findIndex((v) => v === profile); + if (idx === -1) { + throw new Error(`Invalid profile ${profile}`); } - }; - - const selectedCommandTree = selectedProfile ? commandTrees[selectedProfile] : undefined; - - const handleBackToHomepage = () => { - window.open('/?#/cli', "_blank"); - }; - - const handleGenerate = () => { - setShowGenerateDialog(true); - }; - - const handleDraftPowerShell = () => { - setShowDraftPowerShellDialog(true); - }; - - const handleGenerationClose = () => { - setShowGenerateDialog(false); - }; - - const handleDraftPowerShellClose = () => { - setShowDraftPowerShellDialog(false); - }; - - const onProfileChange = React.useCallback((selectedProfile: string) => { - setSelectedProfile(selectedProfile); - }, []); - - const onSelectedProfileTreeUpdate = React.useCallback((updater: ((oldTree: ProfileCommandTree) => ProfileCommandTree) | ProfileCommandTree) => { - setCommandTrees((commandTrees) => { - const selectedCommandTree = commandTrees[selectedProfile!]; - const newTree = typeof updater === 'function' ? updater(selectedCommandTree!) : updater; - return { ...commandTrees, [selectedProfile!]: newTree } - }); - }, [selectedProfile]); - - return ( - - { + return [ + profile, + InitializeCommandTreeByModView( + profile, + modView!.profiles[profile] ?? null, + simpleTree + ), + ]; + }) + ); + + const selectedProfile = profiles.length > 0 ? profiles[0] : undefined; + setProfiles(profiles); + setCommandTrees(commandTrees); + setSelectedProfile(selectedProfile); + setLoading(false); + } catch (err: any) { + console.error(err); + if (err.response?.data?.message) { + const data = err.response!.data!; + setInvalidText(`ResponseError: ${data.message!}`); + } else { + setInvalidText(`Error: ${err}`); + } + } + }; + + const selectedCommandTree = selectedProfile + ? commandTrees[selectedProfile] + : undefined; + + const handleBackToHomepage = () => { + window.open("/?#/cli", "_blank"); + }; + + const handleGenerate = () => { + setShowGenerateDialog(true); + }; + + const handleDraftPowerShell = () => { + setShowDraftPowerShellDialog(true); + }; + + const handleGenerationClose = () => { + setShowGenerateDialog(false); + }; + + const handleDraftPowerShellClose = () => { + setShowDraftPowerShellDialog(false); + }; + + const onProfileChange = React.useCallback((selectedProfile: string) => { + setSelectedProfile(selectedProfile); + }, []); + + const onSelectedProfileTreeUpdate = React.useCallback( + ( + updater: + | ((oldTree: ProfileCommandTree) => ProfileCommandTree) + | ProfileCommandTree + ) => { + setCommandTrees((commandTrees) => { + const selectedCommandTree = commandTrees[selectedProfile!]; + const newTree = + typeof updater === "function" + ? updater(selectedCommandTree!) + : updater; + return { ...commandTrees, [selectedProfile!]: newTree }; + }); + }, + [selectedProfile] + ); + + return ( + + + + + + {selectedProfile !== undefined && ( + - - - - {selectedProfile !== undefined && ( - - )} - - - - {selectedCommandTree !== undefined && ( - - )} - - - {showGenerateDialog && ( - - )} - {!showGenerateDialog && showDraftPowerShellDialog && ( - - )} - theme.zIndex.drawer + 1 }} - open={loading} - > - {invalidText !== undefined ? ( - { - setInvalidText(undefined); - setLoading(false); - }} - > - {invalidText} - - ) : ( - - )} - - - ); + )} + + + + {selectedCommandTree !== undefined && ( + + )} + + + {showGenerateDialog && ( + + )} + {!showGenerateDialog && showDraftPowerShellDialog && ( + + )} + theme.zIndex.drawer + 1 }} + open={loading} + > + {invalidText !== undefined ? ( + { + setInvalidText(undefined); + setLoading(false); + }} + > + {invalidText} + + ) : ( + + )} + + + ); }; function GenerateDialog(props: { - repoName: string; - moduleName: string; - profileCommandTrees: ProfileCommandTrees; - open: boolean; - onClose: (generated: boolean) => void; + repoName: string; + moduleName: string; + profileCommandTrees: ProfileCommandTrees; + open: boolean; + onClose: (generated: boolean) => void; }) { - const [updating, setUpdating] = React.useState(false); - const [invalidText, setInvalidText] = React.useState( - undefined - ); - - const handleClose = () => { - props.onClose(false); + const [updating, setUpdating] = React.useState(false); + const [invalidText, setInvalidText] = React.useState( + undefined + ); + + const handleClose = () => { + props.onClose(false); + }; + + const handleGenerateAll = () => { + const profiles: CLIModViewProfiles = {}; + Object.values(props.profileCommandTrees).forEach((tree) => { + profiles[tree.name] = ExportModViewProfile(tree); + }); + const data = { + name: props.moduleName, + profiles: profiles, }; - const handleGenerateAll = () => { - const profiles: CLIModViewProfiles = {}; - Object.values(props.profileCommandTrees).forEach(tree => { - profiles[tree.name] = ExportModViewProfile(tree); - }) - const data = { - name: props.moduleName, - profiles: profiles, + setUpdating(true); + axios + .put(`/CLI/Az/${props.repoName}/Modules/${props.moduleName}`, data) + .then(() => { + setUpdating(false); + props.onClose(true); + }) + .catch((err) => { + console.error(err); + if (err.response?.data?.message) { + const data = err.response!.data!; + setInvalidText( + `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}` + ); } - - setUpdating(true); - axios - .put( - `/CLI/Az/${props.repoName}/Modules/${props.moduleName}`, - data - ) - .then(() => { - setUpdating(false); - props.onClose(true); - }) - .catch((err) => { - console.error(err); - if (err.response?.data?.message) { - const data = err.response!.data!; - setInvalidText( - `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}` - ); - } - setUpdating(false); - }); + setUpdating(false); + }); + }; + + const handleGenerateModified = () => { + const profiles: CLIModViewProfiles = {}; + Object.values(props.profileCommandTrees).forEach((tree) => { + profiles[tree.name] = ExportModViewProfile(tree); + }); + const data = { + name: props.moduleName, + profiles: profiles, }; - const handleGenerateModified = () => { - const profiles: CLIModViewProfiles = {}; - Object.values(props.profileCommandTrees).forEach(tree => { - profiles[tree.name] = ExportModViewProfile(tree); - }) - const data = { - name: props.moduleName, - profiles: profiles, + setUpdating(true); + axios + .patch(`/CLI/Az/${props.repoName}/Modules/${props.moduleName}`, data) + .then(() => { + setUpdating(false); + props.onClose(true); + }) + .catch((err) => { + console.error(err); + if (err.response?.data?.message) { + const data = err.response!.data!; + setInvalidText( + `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}` + ); } + setUpdating(false); + }); + }; + + return ( + + Generate CLI commands to {props.moduleName} + + {invalidText && ( + + {" "} + {invalidText}{" "} + + )} + + + {updating && ( + + + + )} + {!updating && ( + + + + + + )} + + + ); +} - setUpdating(true); - axios - .patch( - `/CLI/Az/${props.repoName}/Modules/${props.moduleName}`, - data - ) - .then(() => { - setUpdating(false); - props.onClose(true); - }) - .catch((err) => { - console.error(err); - if (err.response?.data?.message) { - const data = err.response!.data!; - setInvalidText( - `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}` - ); - } - setUpdating(false); - }); - } +interface PSModule { + name: string; + folder: string | null; + url: string | null; +} - return ( - - Generate CLI commands to {props.moduleName} - - {invalidText && {invalidText} } - - - {updating && - - - - } - {!updating && - - - - } - - - ); +interface InputType { + inputValue: string; + title: string; } -function DraftPowerShellDialog(props: { - open: boolean; - onClose: (generated: boolean) => void; -}) { +interface PSSketchProfile { + resourceProviders: PSSketchResourceProvider[]; +} - const [loading, setLoading] = React.useState(false); - const [updating, setUpdating] = React.useState(false); - const [invalidText, setInvalidText] = React.useState( - undefined - ); - const [powershellPath, setPowershellPath] = React.useState(undefined); +interface PSSketchResourceProvider { + swagger: string; + resources?: PSSketchResource[]; + selected?: boolean; +} - const handleClose = () => { - props.onClose(false); - }; +interface PSSketchResource { + id: string; + path: string; + subresources?: string[]; +} - const handleDraftPowerShell = () => { - setUpdating(true); - // axios - // .post(`/CLI/Az/${props.repoName}/Modules/${props.moduleName}/DraftPowerShell`) - // .then(() => { - // setUpdating(false); - // props.onClose(true); - // }) +function DraftPowerShellDialog(props: { + repoName: string; + moduleName: string; + profileCommandTrees: ProfileCommandTrees; + open: boolean; + onClose: (generated: boolean) => void; +}) { + const [loading, setLoading] = React.useState(false); + const [updating, setUpdating] = React.useState(false); + const [invalidText, setInvalidText] = React.useState( + undefined + ); + const [selectedModule, setSelectedModule] = React.useState( + null + ); + + const [sketchProfile, setSketchProfile] = + React.useState(null); + const [moduleOptions, setModuleOptions] = React.useState([]); + + const handleClose = () => { + props.onClose(false); + }; + + const filter = createFilterOptions(); + + const handleDraftPowerShell = () => { + setUpdating(true); + // axios + // .post(`/CLI/Az/${props.repoName}/Modules/${props.moduleName}/DraftPowerShell`) + // .then(() => { + // setUpdating(false); + // props.onClose(true); + // }) + }; + + const loadData = React.useCallback(async () => { + setLoading(true); + const cliModViewProfile = ExportModViewProfile( + props.profileCommandTrees["latest"] + ); + try { + let response = await axios.get(`/PS/Powershell/Modules`); + const options = response.data.map((option: any) => { + return { + name: option.name, + folder: option.folder, + url: option.url, + }; + }); + setModuleOptions(options); + response = await axios.post(`/PS/Editor/GenerateSketcheProfile`, { + cliProfile: cliModViewProfile, + }); + let profile: PSSketchProfile = response.data; + profile.resourceProviders.forEach((rp: PSSketchResourceProvider) => { + rp.selected = true; + }); + setSketchProfile(profile); + setLoading(false); + } catch (err: any) { + console.error(err); + if (err.response?.data?.message) { + const data = err.response!.data!; + setInvalidText( + `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}` + ); + } else { + setInvalidText(err.message); + } + setLoading(false); } + }, [props.profileCommandTrees]); - React.useEffect(() => { - if (props.open) { - setLoading(true); - // call /PS/PowerShell/Path api to get the path of the PowerShell script - axios.get(`/PS/Powershell/Path`).then(res => { - setPowershellPath(res.data.path || undefined); - setLoading(false); - }).catch(err => { - console.error(err); - setLoading(false); - }); - } else { - setLoading(false); - } - }, [props.open]); - - return - Draft PowerShell Generation from CLI - - {invalidText && {invalidText} } - {loading && } - {!loading && + React.useEffect(() => { + if (props.open) { + loadData(); + } else { + setLoading(false); + } + }, [props.open]); + + const handleResourceProviderChange = ( + event: React.ChangeEvent, + index: number + ) => { + setSketchProfile((prevProfile) => { + const newResourceProviders = [...prevProfile!.resourceProviders]; + newResourceProviders[index].selected = event.target.checked; + return { ...prevProfile!, resourceProviders: newResourceProviders }; + }); + }; + + const submitable = + selectedModule !== null && + sketchProfile?.resourceProviders.some((rp) => rp.selected); + + // TODO: need user select the resource providers to generation as click boxes + return ( + + Draft PowerShell Generation from CLI + + {invalidText && ( + + {" "} + {invalidText}{" "} + + )} + {loading && } + {!loading && ( + + { + if (typeof newValue === "string") { + setSelectedModule({ + name: newValue, + folder: null, + url: null, + }); + } else if (newValue && newValue.inputValue) { + setSelectedModule({ + name: newValue.inputValue, + folder: null, + url: null, + }); + } else { + setSelectedModule(newValue as PSModule); + } + }} + filterOptions={(options, params: any) => { + const filtered = filter(options, params); + if ( + params.inputValue !== "" && + -1 === options.findIndex((e) => e.name === params.inputValue) + ) { + filtered.push({ + inputValue: params.inputValue, + title: `Create "${params.inputValue}"`, + }); + } + return filtered; + }} + getOptionLabel={(option) => { + if (typeof option === "string") { + return option; + } + if (option.title) { + return option.title; + } + return option.name; + }} + renderOption={(props, option) => ( + + {option && option.title ? option.title : option.name} + + )} + selectOnFocus + clearOnBlur + renderInput={(params) => ( setPowershellPath(e.target.value)} - InputProps={{ - endAdornment: ( - - { - const dirHandler = await (window as any).showDirectoryPicker(); - const path = await dirHandler.resolve(); - setPowershellPath(path); - }} - > - - - - ) - }} + {...params} + label="PowerShell Module" + inputProps={{ + ...params.inputProps, + placeholder: + "The folder to generate the Autorest code, e.g. ServiceName/ServiceName.Autorest", + autoComplete: "new-password", // disable autocomplete and autofill + }} /> - } - - - {updating && - - - - } - {!updating && - - - } - + )} + > + Resource Providers + {sketchProfile && ( + + {sketchProfile.resourceProviders.map((rp, index) => ( + handleResourceProviderChange(e, index)} + /> + } + label={rp.swagger} + /> + ))} + + )} + + )} + + + {updating && ( + + + + )} + {!updating && ( + + + + + )} + + ); } const CLIModuleGeneratorWrapper = (props: any) => { - const params = useParams(); - return -} + const params = useParams(); + return ; +}; -export type { CLISpecsCommandGroup, CLISpecsCommand, CLISpecsSimpleCommandTree, CLISpecsSimpleCommandGroup, CLISpecsSimpleCommand }; +export type { + CLISpecsCommandGroup, + CLISpecsCommand, + CLISpecsSimpleCommandTree, + CLISpecsSimpleCommandGroup, + CLISpecsSimpleCommand, +}; export { CLIModuleGeneratorWrapper as CLIModuleGenerator }; From 538a430e7b5d9f43c17409aefd76bafe201127a3 Mon Sep 17 00:00:00 2001 From: kai ru Date: Thu, 23 Jan 2025 14:02:56 +0800 Subject: [PATCH 18/18] add ps profile module --- src/aaz_dev/ps/model/sketch/_profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/aaz_dev/ps/model/sketch/_profile.py b/src/aaz_dev/ps/model/sketch/_profile.py index 862354fe..9404f05a 100644 --- a/src/aaz_dev/ps/model/sketch/_profile.py +++ b/src/aaz_dev/ps/model/sketch/_profile.py @@ -12,5 +12,3 @@ class PSSketchProfile(Model): class Options: serialize_when_none = False - -