Add ps module

src/aaz_dev/app/app.py

@@ -27,12 +27,13 @@ def invalid_api_usage(e):
         return jsonify(e.to_dict()), e.status_code
 
     # register url converters
-    from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor
+    from .url_converters import Base64Converter, NameConverter, NameWithCapitalConverter, NamesPathConverter, ListPathConvertor, PSNamesPathConverter
     app.url_map.converters['base64'] = Base64Converter
     app.url_map.converters['name'] = NameConverter
     app.url_map.converters['Name'] = NameWithCapitalConverter
     app.url_map.converters['names_path'] = NamesPathConverter
     app.url_map.converters['list_path'] = ListPathConvertor
+    app.url_map.converters['PSNamesPath'] = PSNamesPathConverter
 
     # register routes of swagger module
     from swagger.api import register_blueprints
@@ -46,6 +47,10 @@ def invalid_api_usage(e):
     from cli.api import register_blueprints
     register_blueprints(app)
 
+    # register routes of ps module
+    from ps.api import register_blueprints
+    register_blueprints(app)
+
     return app
 
 

src/aaz_dev/app/run.py

@@ -85,6 +85,14 @@ def is_port_in_use(host, port):
     expose_value=False,
     help="The local path of azure-cli-extension repo. Official repo is https://github.com/Azure/azure-cli-extensions"
 )
+@click.option(
+    "--powershell-path", '--ps',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    default=Config.POWERSHELL_PATH,
+    callback=Config.validate_and_setup_powershell_path,
+    expose_value=False,
+    help="The local path of azure-powershell repo."
+)
 @click.option(
     "--workspaces-path", '-w',
     type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
@@ -94,6 +102,15 @@ def is_port_in_use(host, port):
     expose_value=False,
     help="The folder to load and save workspaces."
 )
+@click.option(
+    "--sketch-path", '-k',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    default=Config.AAZ_DEV_SKETCH_FOLDER,
+    required=not Config.AAZ_DEV_SKETCH_FOLDER,
+    callback=Config.validate_and_setup_aaz_dev_sketch_folder,
+    expose_value=False,
+    help="The folder to load and save sketches for PowerShell generation."
+)
 @click.option(
     "--reload/--no-reload",
     default=None,

src/aaz_dev/app/tests/common.py

@@ -11,8 +11,10 @@ class ApiTestCase(TestCase):
 
     def __init__(self, *args, **kwargs):
         self.cleanup_dev_folder()
-        Config.AAZ_PATH = self.AAZ_FOLDER
-        Config.AAZ_DEV_FOLDER = self.AAZ_DEV_FOLDER
+        if self.AAZ_FOLDER is not None:
+            Config.AAZ_PATH = self.AAZ_FOLDER
+        if self.AAZ_DEV_FOLDER is not None:
+            Config.AAZ_DEV_FOLDER = self.AAZ_DEV_FOLDER
         Config.AAZ_DEV_WORKSPACE_FOLDER = os.path.join(self.AAZ_DEV_FOLDER, 'workspaces')
         super().__init__(*args, **kwargs)
         self.app = create_app()
@@ -22,9 +24,12 @@ def __init__(self, *args, **kwargs):
     def cleanup_dev_folder(self):
         if os.path.exists(self.AAZ_DEV_FOLDER):
             shutil.rmtree(self.AAZ_DEV_FOLDER)
-        if os.path.exists(self.AAZ_FOLDER):
+        if self.AAZ_FOLDER is not None and os.path.exists(self.AAZ_FOLDER):
             shutil.rmtree(self.AAZ_FOLDER)
 
     def setUp(self):
-        os.makedirs(self.AAZ_FOLDER, exist_ok=True)
+        if self.AAZ_FOLDER is not None:
+            os.makedirs(self.AAZ_FOLDER, exist_ok=True)
+        if self.AAZ_DEV_FOLDER is not None:
+            os.makedirs(self.AAZ_DEV_FOLDER, exist_ok=True)
 

src/aaz_dev/app/url_converters.py

@@ -32,7 +32,6 @@ def to_url(self, values):
             return super(NamesPathConverter, self).to_url(values)
         return '/'.join(super(NamesPathConverter, self).to_url(value) for value in values)
 
-
 class ListPathConvertor(PathConverter):
 
     def to_python(self, value):
@@ -43,5 +42,17 @@ def to_url(self, values):
             return super(ListPathConvertor, self).to_url(values)
         return '/'.join(super(ListPathConvertor, self).to_url(value) for value in values)
 
+class PSNamesPathConverter(PathConverter):
+    regex = r"([A-Z][a-zA-Z0-9]*)/([A-Z][a-zA-Z0-9]*\.Autorest)"
+    weight = 200
+
+    def to_python(self, value):
+        return value.split('/')
+
+    def to_url(self, values):
+        if isinstance(values, str):
+            return super(PSNamesPathConverter, self).to_url(values)
+        return '/'.join(super(PSNamesPathConverter, self).to_url(value) for value in values)
+
 
-__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor"]
+__all__ = ["Base64Converter", "NameConverter", "NamesPathConverter", "ListPathConvertor", "PSNamesPathConverter"]

src/aaz_dev/command/api/_cmds.py

@@ -167,40 +167,75 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None):
     help="Path of `aaz` repository."
 )
 def verify():
-    def verify_command(file_path, node):
+    def verify_resource(model, path):
+        if "commandGroups" not in model:
+            return
+
+        for grp in model["commandGroups"]:
+            base_path = os.path.join(path, *grp["name"].split())
+            if not os.path.exists(base_path):
+                raise FileNotFoundError(base_path)
+
+            for cmd in grp.get("commands", []):
+                file_path = os.path.join(base_path, f"_{cmd['name']}.md")
+                if not os.path.isfile(file_path):
+                    raise FileNotFoundError(file_path)
+
+            verify_resource(grp, base_path)
+
+    def verify_command(file_path):
         with open(file_path, "r", encoding="utf-8") as fp:
             content = fp.read()
 
+        base_path = os.path.dirname(file_path)
+        curr_grp = " ".join(os.path.relpath(base_path, aaz.commands_folder).split(os.sep))
+        curr_cmd = os.path.splitext(os.path.basename(file_path))[0][1:]
+
         paths = re.findall(r"]\(([^)]+)\)", content)
         for path in paths:
             json_path = os.path.join(Config.AAZ_PATH, os.path.splitext(path)[0][1:] + ".json")
             json_path = os.path.normpath(json_path)
+
+            if json_path in model_set:
+                continue
+
             if not os.path.exists(json_path):
                 raise Exception(f"{json_path} defined in {file_path} is missing.")
 
             with open(json_path, "r", encoding="utf-8", errors="ignore") as fp:
                 model = json.load(fp)
-            group, command = " ".join(node.names[:-1]), node.names[-1]
-            for g in model["commandGroups"]:
-                if g["name"] == group:
-                    if not any(cmd["name"] == command for cmd in g["commands"]):
-                        raise Exception(f"There is no {command} command info in {json_path}.")
 
-                    break
+            try:
+                verify_resource(model, aaz.commands_folder)
 
-            model_set.add(json_path)
+            except FileNotFoundError as e:
+                raise Exception(f"Cannot find {e} defined in {json_path}.")
 
-        tmpl = get_templates()["command"]
-        if not tmpl.render(command=node) == content:
-            raise Exception(f"{file_path} cannot be rendered correctly.")
+            target = curr_grp
+            while target:
+                try:
+                    for grp in model["commandGroups"]:
+                        if target.startswith(grp["name"]):
+                            target = target[len(grp["name"]):].strip()
+                            model = grp
+
+                            break
+
+                except KeyError:
+                    raise Exception(f"{curr_grp} has no corresponding definition in {json_path}.")
+
+            commands = model["commands"]
+            if not any(cmd["name"] == curr_cmd for cmd in commands):
+                raise Exception(f"There is no {curr_cmd} command info in {json_path}.")
+
+            model_set.add(json_path)
 
     model_set = set()
     aaz = AAZSpecsManager()
-    stack = [(aaz.commands_folder, aaz.tree.root)]  # root nodes
+    stack = [aaz.commands_folder]
 
     while stack:
-        curr_path, curr_node = stack.pop()
-        logger.info(f"Checking {curr_path}")
+        curr_path = stack.pop()
         if os.path.isdir(curr_path):
             readme_path = os.path.join(curr_path, "readme.md")
             if not os.path.exists(readme_path):
@@ -227,13 +262,9 @@ def verify_command(file_path, node):
                         diff = cmd_set - items or items - cmd_set
                         raise Exception(f"Command info {diff} doesn't match in {readme_path}.")
 
-                    groups = set(curr_node.commands.keys())
-                    if groups != items:
-                        diff = groups - items or items - groups
-                        raise Exception(f"Command info {diff} in tree.json doesn't match in {readme_path}.")
-
                     for file in files:
-                        verify_command(os.path.join(curr_path, file), curr_node.commands[file[1:-3]])
+                        verify_command(os.path.join(curr_path, file))
+
                 else:
                     if len(items) != len(set(items)):
                         raise Exception(f"{readme_path} has duplicate command group names.")
@@ -245,15 +276,10 @@ def verify_command(file_path, node):
                         diff = folders - items or items - folders
                         raise Exception(f"Command group info {diff} doesn't match in {readme_path}.")
 
-                    groups = set(curr_node.command_groups.keys())
-                    if groups != set(items):
-                        diff = groups - items or items - groups
-                        raise Exception(f"Command group info {diff} in tree.json doesn't match in {readme_path}.")
-
                     for folder in folders:
-                        stack.append((os.path.join(curr_path, folder), curr_node.command_groups[folder]))
+                        stack.append(os.path.join(curr_path, folder))
 
-    for root, dirs, files in os.walk(aaz.resources_folder):
+    for root, _, files in os.walk(aaz.resources_folder):
         for file in files:
             if not file.endswith(".json") or file.startswith("client"):  # support data-plane
                 continue

src/aaz_dev/ps/api/__init__.py

@@ -0,0 +1,7 @@
+
+def register_blueprints(app):
+    from . import _cmds, autorest, powershell, editor
+    app.register_blueprint(_cmds.bp)
+    app.register_blueprint(autorest.bp)
+    app.register_blueprint(powershell.bp)
+    app.register_blueprint(editor.bp)

src/aaz_dev/ps/api/_cmds.py

@@ -0,0 +1,140 @@
+import click
+import logging
+from flask import Blueprint
+import sys
+import os
+import subprocess
+
+from utils.config import Config
+
+logger = logging.getLogger('backend')
+
+bp = Blueprint('ps-cmds', __name__, url_prefix='/PS/CMDs', cli_group="ps")
+bp.cli.short_help = "Manage powershell commands."
+
+
+@bp.cli.command("generate-powershell", short_help="Generate powershell code based on selected azure cli module.")
+@click.option(
+    "--aaz-path", '-a',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    default=Config.AAZ_PATH,
+    required=not Config.AAZ_PATH,
+    callback=Config.validate_and_setup_aaz_path,
+    expose_value=False,
+    help="The local path of aaz repo."
+)
+@click.option(
+    "--cli-path", '-c',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    callback=Config.validate_and_setup_cli_path,
+    help="The local path of azure-cli repo. Only required when generate from azure-cli module."
+)
+@click.option(
+    "--cli-extension-path", '-e',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    callback=Config.validate_and_setup_cli_extension_path,
+    help="The local path of azure-cli-extension repo. Only required when generate from azure-cli extension."
+)
+@click.option(
+    "--powershell-path", '--ps',
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True),
+    callback=Config.validate_and_setup_powershell_path,
+    help="The local path of azure-powershell repo."
+)
+@click.option(
+    "--extension-or-module-name", '--name',
+    required=True,
+    help="Name of the module in azure-cli or the extension in azure-cli-extensions"
+)
+@click.option(
+    "--swagger-path", '-s',
+    type=click.Path(file_okay=False, dir_okay=True, readable=True, resolve_path=True),
+    default=Config.SWAGGER_PATH,
+    required=not Config.SWAGGER_PATH,
+    callback=Config.validate_and_setup_swagger_path,
+    expose_value=False,
+    help="The local path of azure-rest-api-specs repo. Official repo is https://github.com/Azure/azure-rest-api-specs"
+)
+def generate_powershell(extension_or_module_name, cli_path=None, cli_extension_path=None, powershell_path=None):
+    from ps.controller.autorest_configuration_generator import PSAutoRestConfigurationGenerator
+    from cli.controller.az_module_manager import AzMainManager, AzExtensionManager
+    from ps.templates import get_templates
+
+    # Module path in azure-powershell repo
+
+    powershell_path = os.path.join(powershell_path, "src")
+    if not os.path.exists(powershell_path):
+        logger.error(f"Path `{powershell_path}` not exist")
+        sys.exit(1)
+
+    if cli_path is not None:
+        assert Config.CLI_PATH is not None
+        manager = AzMainManager()
+    else:
+        assert cli_extension_path is not None
+        assert Config.CLI_EXTENSION_PATH is not None
+        manager = AzExtensionManager()
+
+    if not manager.has_module(extension_or_module_name):
+        logger.error(f"Cannot find module or extension `{extension_or_module_name}`")
+        sys.exit(1)
+
+    # generate README.md for powershell from CLI, ex, for Oracle, README.md should be generated in src/Oracle/Oracle.Autorest/README.md in azure-powershell repo
+    ps_generator = PSAutoRestConfigurationGenerator(manager, extension_or_module_name)
+    ps_cfg = ps_generator.generate_config()
+
+    autorest_module_path = os.path.join(powershell_path, ps_cfg.module_name, f"{ps_cfg.module_name}.Autorest")
+    if not os.path.exists(autorest_module_path):
+        os.makedirs(autorest_module_path)
+    readme_file = os.path.join(autorest_module_path, "README.md")
+    if os.path.exists(readme_file):
+        # read until to the "### AutoRest Configuration"
+        with open(readme_file, "r") as f:
+            lines = f.readlines()
+            for i, line in enumerate(lines):
+                if line.startswith("### AutoRest Configuration"):
+                    lines = lines[:i]
+                    break
+    else:
+        lines = []
+
+    tmpl = get_templates()['autorest']['configuration']
+    data = tmpl.render(cfg=ps_cfg)
+    lines.append(data)
+    with open(readme_file, "w") as f:
+        f.writelines(lines)
+
+    print(f"Generated {readme_file}")
+    # Generate and build PowerShell module from the README.md file generated above
+    print("Start to generate the PowerShell module from the README.md file in " + autorest_module_path)
+
+    # Execute autorest to generate the PowerShell module
+    original_cwd = os.getcwd()
+    os.chdir(autorest_module_path)
+    exit_code = os.system("pwsh -Command autorest")
+
+     # Print the output of the generation
+    if (exit_code != 0):
+        print("Failed to generate the module")
+        os.chdir(original_cwd)
+        sys.exit(1)
+    else:
+        print("Code generation succeeded.")
+        # print(result.stdout)
+
+    os.chdir(original_cwd)
+    # Execute autorest to generate the PowerShell module
+    print("Start to build the generated PowerShell module")
+    result = subprocess.run(
+        ["pwsh", "-File", 'build-module.ps1'],
+        capture_output=True,
+        text=True,
+        cwd=autorest_module_path
+    )
+
+    if (result.returncode != 0):
+        print("Failed to build the module, please see following output for details:")
+        print(result.stderr)
+        sys.exit(1)
+    else:
+        print("Module build succeeds, and you may run the generated module by executing the following command: `./run-module.ps1` in " + autorest_module_path)