From 5eb68686c9e888980c1312b6af3af1aac8626c9c Mon Sep 17 00:00:00 2001 From: necusjz Date: Fri, 6 Dec 2024 14:53:22 +0800 Subject: [PATCH 1/5] perf: no need to verify tree.json --- src/aaz_dev/command/api/_cmds.py | 37 ++++++-------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index d4d9e100..1e902317 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -167,7 +167,7 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): help="Path of `aaz` repository." ) def verify(): - def verify_command(file_path, node): + def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() @@ -175,32 +175,18 @@ def verify_command(file_path, node): for path in paths: json_path = os.path.join(Config.AAZ_PATH, os.path.splitext(path)[0][1:] + ".json") json_path = os.path.normpath(json_path) + if not os.path.exists(json_path): raise Exception(f"{json_path} defined in {file_path} is missing.") - with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: - model = json.load(fp) - group, command = " ".join(node.names[:-1]), node.names[-1] - for g in model["commandGroups"]: - if g["name"] == group: - if not any(cmd["name"] == command for cmd in g["commands"]): - raise Exception(f"There is no {command} command info in {json_path}.") - - break - model_set.add(json_path) - tmpl = get_templates()["command"] - if not tmpl.render(command=node) == content: - raise Exception(f"{file_path} cannot be rendered correctly.") - model_set = set() aaz = AAZSpecsManager() - stack = [(aaz.commands_folder, aaz.tree.root)] # root nodes + stack = [aaz.commands_folder] while stack: - curr_path, curr_node = stack.pop() - logger.info(f"Checking {curr_path}") + curr_path = stack.pop() if os.path.isdir(curr_path): readme_path = os.path.join(curr_path, "readme.md") if not os.path.exists(readme_path): @@ -227,13 +213,9 @@ def verify_command(file_path, node): diff = cmd_set - items or items - cmd_set raise Exception(f"Command info {diff} doesn't match in {readme_path}.") - groups = set(curr_node.commands.keys()) - if groups != items: - diff = groups - items or items - groups - raise Exception(f"Command info {diff} in tree.json doesn't match in {readme_path}.") - for file in files: - verify_command(os.path.join(curr_path, file), curr_node.commands[file[1:-3]]) + verify_command(os.path.join(curr_path, file)) + else: if len(items) != len(set(items)): raise Exception(f"{readme_path} has duplicate command group names.") @@ -245,13 +227,8 @@ def verify_command(file_path, node): diff = folders - items or items - folders raise Exception(f"Command group info {diff} doesn't match in {readme_path}.") - groups = set(curr_node.command_groups.keys()) - if groups != set(items): - diff = groups - items or items - groups - raise Exception(f"Command group info {diff} in tree.json doesn't match in {readme_path}.") - for folder in folders: - stack.append((os.path.join(curr_path, folder), curr_node.command_groups[folder])) + stack.append(os.path.join(curr_path, folder)) for root, dirs, files in os.walk(aaz.resources_folder): for file in files: From 942cd7d9e16868653dd1d461cc1d7c3145893ebc Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 16 Dec 2024 16:28:56 +0800 Subject: [PATCH 2/5] feat: verify json to commands --- src/aaz_dev/command/api/_cmds.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 1e902317..0da48997 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -171,14 +171,37 @@ def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() + folder = os.path.dirname(file_path) + curr_cmd = os.path.splitext(os.path.basename(file_path))[0][1:] + curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) + + # _command_name.md -> command_name + cmd_set = set(map(lambda x: x[1:-3], [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))])) + paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: json_path = os.path.join(Config.AAZ_PATH, os.path.splitext(path)[0][1:] + ".json") json_path = os.path.normpath(json_path) + if json_path in model_set: + continue + if not os.path.exists(json_path): raise Exception(f"{json_path} defined in {file_path} is missing.") + with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: + model = json.load(fp) + + for grp in model["commandGroups"]: + if grp["name"] == curr_grp: + if not any(cmd["name"] == curr_cmd for cmd in grp["commands"]): + raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + + if any(cmd["name"] not in cmd_set for cmd in grp["commands"]): + raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") + + break + model_set.add(json_path) model_set = set() From c927001c5547574a9b88d77a94482e6d68988a57 Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 17 Dec 2024 16:26:42 +0800 Subject: [PATCH 3/5] fix: support nested command groups --- src/aaz_dev/command/api/_cmds.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 0da48997..2f59ed26 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -192,15 +192,24 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) - for grp in model["commandGroups"]: - if grp["name"] == curr_grp: - if not any(cmd["name"] == curr_cmd for cmd in grp["commands"]): - raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + group = [] + while True: + try: + group.append(model["commandGroups"][0]["name"]) + if " ".join(group) == curr_grp: + break - if any(cmd["name"] not in cmd_set for cmd in grp["commands"]): - raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") + model = model["commandGroups"][0] - break + except KeyError: + raise Exception(f"{curr_grp} {curr_cmd} is redundant.") + + commands = model["commandGroups"][0]["commands"] + if not any(cmd["name"] == curr_cmd for cmd in commands): + raise Exception(f"There is no {curr_cmd} command info in {json_path}.") + + if any(cmd["name"] not in cmd_set for cmd in commands): + raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") model_set.add(json_path) From c5b222f052b47933072449d4d477de3dd5ac2b72 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 18 Dec 2024 14:10:05 +0800 Subject: [PATCH 4/5] fix: group may contain many subgroups --- src/aaz_dev/command/api/_cmds.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 2f59ed26..104bd3dd 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -176,7 +176,7 @@ def verify_command(file_path): curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) # _command_name.md -> command_name - cmd_set = set(map(lambda x: x[1:-3], [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))])) + cmd_set = set(map(lambda x: x[1:-3], [i for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i))])) paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: @@ -192,19 +192,20 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) - group = [] - while True: + target = curr_grp + while target: try: - group.append(model["commandGroups"][0]["name"]) - if " ".join(group) == curr_grp: - break + for grp in model["commandGroups"]: + if target.startswith(grp["name"]): + target = target[len(grp["name"]):].strip() + model = grp - model = model["commandGroups"][0] + break except KeyError: raise Exception(f"{curr_grp} {curr_cmd} is redundant.") - commands = model["commandGroups"][0]["commands"] + commands = model["commands"] if not any(cmd["name"] == curr_cmd for cmd in commands): raise Exception(f"There is no {curr_cmd} command info in {json_path}.") From 8de427fd35ce6880bbaab66c86aa9ce67583fb72 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 19 Dec 2024 22:16:21 +0800 Subject: [PATCH 5/5] feat: add verify_resource --- src/aaz_dev/command/api/_cmds.py | 36 +++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 104bd3dd..c3c239f4 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -167,16 +167,29 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): help="Path of `aaz` repository." ) def verify(): + def verify_resource(model, path): + if "commandGroups" not in model: + return + + for grp in model["commandGroups"]: + base_path = os.path.join(path, *grp["name"].split()) + if not os.path.exists(base_path): + raise FileNotFoundError(base_path) + + for cmd in grp.get("commands", []): + file_path = os.path.join(base_path, f"_{cmd['name']}.md") + if not os.path.isfile(file_path): + raise FileNotFoundError(file_path) + + verify_resource(grp, base_path) + def verify_command(file_path): with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() - folder = os.path.dirname(file_path) + base_path = os.path.dirname(file_path) + curr_grp = " ".join(os.path.relpath(base_path, aaz.commands_folder).split(os.sep)) curr_cmd = os.path.splitext(os.path.basename(file_path))[0][1:] - curr_grp = " ".join(os.path.relpath(folder, aaz.commands_folder).split(os.sep)) - - # _command_name.md -> command_name - cmd_set = set(map(lambda x: x[1:-3], [i for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i))])) paths = re.findall(r"]\(([^)]+)\)", content) for path in paths: @@ -192,6 +205,12 @@ def verify_command(file_path): with open(json_path, "r", encoding="utf-8", errors="ignore") as fp: model = json.load(fp) + try: + verify_resource(model, aaz.commands_folder) + + except FileNotFoundError as e: + raise Exception(f"Cannot find {e} defined in {json_path}.") + target = curr_grp while target: try: @@ -203,15 +222,12 @@ def verify_command(file_path): break except KeyError: - raise Exception(f"{curr_grp} {curr_cmd} is redundant.") + raise Exception(f"{curr_grp} has no corresponding definition in {json_path}.") commands = model["commands"] if not any(cmd["name"] == curr_cmd for cmd in commands): raise Exception(f"There is no {curr_cmd} command info in {json_path}.") - if any(cmd["name"] not in cmd_set for cmd in commands): - raise Exception(f"{curr_grp} defined in {json_path} has command that doesn't exist.") - model_set.add(json_path) model_set = set() @@ -263,7 +279,7 @@ def verify_command(file_path): for folder in folders: stack.append(os.path.join(curr_path, folder)) - for root, dirs, files in os.walk(aaz.resources_folder): + for root, _, files in os.walk(aaz.resources_folder): for file in files: if not file.endswith(".json") or file.startswith("client"): # support data-plane continue