From 9080ab5deefbe735b7de6b79ed4143a5e6e281ec Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 19:43:13 +0200 Subject: [PATCH 01/10] change: remove DFD from default config it is inserted by dfd_extraction.py --- README.md | 1 - code2DFD.py | 2 +- config/config.ini | 2 -- core/dfd_extraction.py | 1 + 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d1b28d2..66f8a54 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ The config file needs to specify the following sections and parameters: - `url`: the full URL of the repository to clone from (may be local path) - `local_path`: local directory to clone the repository to (without the repository name itself) - Technology profiles: same as in [example config](config/config.ini) -- DFD: empty section - Analysis Settings (optional) - `development_mode`: boolean, turns on development mode - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` diff --git a/code2DFD.py b/code2DFD.py index b2085a9..4f2dac2 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -11,7 +11,7 @@ from output_generators.logger import logger import tmp.tmp as tmp -CONFIG_SECTIONS = ["Analysis Settings", "Repository", "Technology Profiles", "DFD"] +CONFIG_SECTIONS = ["Analysis Settings", "Repository", "Technology Profiles"] COMMUNICATIONS_TECH_LIST = '[("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"),\ ("FeignClient", "fgn"), ("Implicit Connections", "imp"),\ ("Database Connections", "dbc"), ("HTML", "html"),\ diff --git a/config/config.ini b/config/config.ini index 6ad38ed..e0bd762 100644 --- a/config/config.ini +++ b/config/config.ini @@ -24,5 +24,3 @@ communication_techs_list = [("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplat development_mode = False ; commit for apssouza22/java-microservice ;commit = 056414c4c938e536f467a3f37532194b860d96a3 - -[DFD] diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index fe0d9a9..adc1d03 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -57,6 +57,7 @@ def perform_analysis(): """ local_path = tmp.tmp_config.get("Repository", "local_path") url_path = tmp.tmp_config.get("Repository", "url") + tmp.tmp_config.add_section("DFD") os.makedirs(local_path, exist_ok=True) repository = Repository(path_to_repo=url_path, clone_repo_to=local_path) From 36c071bbcb583f757d5624a37876570589eee66e Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 19:46:50 +0200 Subject: [PATCH 02/10] change: `commit` is part of `Repository` arguments --- README.md | 2 +- code2DFD.py | 12 ++++++------ config/config.ini | 3 +-- core/dfd_extraction.py | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 66f8a54..cbee549 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,10 @@ The config file needs to specify the following sections and parameters: - `path`: `organization/repository` part of GitHub URL - `url`: the full URL of the repository to clone from (may be local path) - `local_path`: local directory to clone the repository to (without the repository name itself) + - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` - Technology profiles: same as in [example config](config/config.ini) - Analysis Settings (optional) - `development_mode`: boolean, turns on development mode - - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` It is possible to provide these parameters also by command line, see `python3 code2DFD.py --help` for exact usage diff --git a/code2DFD.py b/code2DFD.py index 4f2dac2..e33ba0d 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -43,7 +43,7 @@ def api_invocation(url: str, commit: str) -> dict: tmp.tmp_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) if commit is not None: - tmp.tmp_config.set("Analysis Settings", "commit", commit) + tmp.tmp_config.set("Repository", "commit", commit) # Call extraction codeable_models, traceability = perform_analysis() @@ -69,8 +69,8 @@ def cli_invocation(): repository.add_argument("--repo_url", type=str, help="URL to clone the repository from (might be local path)") repository.add_argument("--repo_local_path", type=str, help="Location to clone repository to (default: 'analysed_repositories' in CWD)") repository.add_argument("--github_handle", type=str, help="Handle of a GitHub repository containing the application to be analyzed") + repository.add_argument("--commit", type=str, help="Analyze repository at this commit") settings = parser.add_argument_group("Analysis Settings", "Parameters for additional analysis settings") - settings.add_argument("--commit", type=str, help="Analyze repository at this commit") settings.add_argument("--development_mode", action='store_true', help="Switch on development mode") args = parser.parse_args() @@ -106,10 +106,10 @@ def cli_invocation(): if args.commit is not None: commit = args.commit[:7] - tmp.tmp_config.set("Analysis Settings", "commit", commit) - elif tmp.tmp_config.has_option("Analysis Settings", "commit"): - commit = tmp.tmp_config.get("Analysis Settings", "commit")[:7] - tmp.tmp_config.set("Analysis Settings", "commit", commit) + tmp.tmp_config.set("Repository", "commit", commit) + elif tmp.tmp_config.has_option("Repository", "commit"): + commit = tmp.tmp_config.get("Repository", "commit")[:7] + tmp.tmp_config.set("Repository", "commit", commit) perform_analysis() diff --git a/config/config.ini b/config/config.ini index e0bd762..5b56350 100644 --- a/config/config.ini +++ b/config/config.ini @@ -1,5 +1,6 @@ [Repository] url = https://github.com/apssouza22/java-microservice +commit = 056414c4c938e536f467a3f37532194b860d96a3 ;url = https://github.com/callistaenterprise/blog-microservices ;url = https://github.com/fernandoabcampos/spring-netflix-oss-microservices ;url = https://github.com/georgwittberger/apache-spring-boot-microservice-example @@ -22,5 +23,3 @@ communication_techs_list = [("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplat [Analysis Settings] development_mode = False -; commit for apssouza22/java-microservice -;commit = 056414c4c938e536f467a3f37532194b860d96a3 diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index adc1d03..018cda8 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -64,8 +64,8 @@ def perform_analysis(): with repository._prep_repo(url_path) as git_repo: tmp.tmp_config.set("Repository", "local_path", str(git_repo.path)) head = git_repo.get_head().hash[:7] - if tmp.tmp_config.has_option("Analysis Settings", "commit"): - commit = tmp.tmp_config.get("Analysis Settings", "commit") + if tmp.tmp_config.has_option("Repository", "commit"): + commit = tmp.tmp_config.get("Repository", "commit") else: commit = head repo_name = git_repo.project_name From 45a72030ef004ea19e539bda437a9069323a16dd Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 19:52:55 +0200 Subject: [PATCH 03/10] change: remove Communications_tech_list from config It is now part of technology_switch.py --- README.md | 1 - code2DFD.py | 7 +------ config/config.ini | 3 --- core/technology_switch.py | 9 ++++++--- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index cbee549..1ee0cbb 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ The config file needs to specify the following sections and parameters: - `url`: the full URL of the repository to clone from (may be local path) - `local_path`: local directory to clone the repository to (without the repository name itself) - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` -- Technology profiles: same as in [example config](config/config.ini) - Analysis Settings (optional) - `development_mode`: boolean, turns on development mode diff --git a/code2DFD.py b/code2DFD.py index e33ba0d..acc3f0e 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -11,16 +11,11 @@ from output_generators.logger import logger import tmp.tmp as tmp -CONFIG_SECTIONS = ["Analysis Settings", "Repository", "Technology Profiles"] -COMMUNICATIONS_TECH_LIST = '[("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"),\ - ("FeignClient", "fgn"), ("Implicit Connections", "imp"),\ - ("Database Connections", "dbc"), ("HTML", "html"),\ - ("Docker-Compose", "dcm")]' +CONFIG_SECTIONS = ["Analysis Settings", "Repository"] DEFAULT_CONFIG = ConfigParser() for section in CONFIG_SECTIONS: DEFAULT_CONFIG.add_section(section) DEFAULT_CONFIG.set("Analysis Settings", "development_mode", "False") -DEFAULT_CONFIG.set("Technology Profiles", "communication_techs_list", COMMUNICATIONS_TECH_LIST) def api_invocation(url: str, commit: str) -> dict: diff --git a/config/config.ini b/config/config.ini index 5b56350..c93564d 100644 --- a/config/config.ini +++ b/config/config.ini @@ -18,8 +18,5 @@ commit = 056414c4c938e536f467a3f37532194b860d96a3 ;url = https://github.com/rohitghatol/spring-boot-microservices ;url = https://github.com/yidongnan/spring-cloud-netflix-example -[Technology Profiles] -communication_techs_list = [("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"), ("FeignClient", "fgn"), ("Implicit Connections", "imp"), ("Database Connections", "dbc"), ("HTML", "html"), ("Docker-Compose", "dcm")] - [Analysis Settings] development_mode = False diff --git a/core/technology_switch.py b/core/technology_switch.py index b1f3617..e6f442b 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -13,6 +13,10 @@ import technology_specific_extractors.resttemplate.rst_entry as rst import tmp.tmp as tmp +COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, + "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, + "Docker-Compose": dcm} + def get_microservices(dfd) -> dict: """Calls get_microservices from correct container technology or returns existing list. @@ -38,9 +42,8 @@ def get_information_flows(dfd) -> dict: return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) else: logger.info("Information flows not set yet, start extraction") - communication_techs_list = ast.literal_eval(tmp.tmp_config["Technology Profiles"]["communication_techs_list"]) - for com_tech in communication_techs_list: - eval(com_tech[1]).set_information_flows(dfd) + for func in COMMUNICATIONS_TECH_LIST.values(): + func.set_information_flows(dfd) if tmp.tmp_config.has_option("DFD", "information_flows"): return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) From 51d44fe3b34f16b867714a0f7acbde2a97cc2c77 Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 20:15:09 +0200 Subject: [PATCH 04/10] refactor: tmp/tmp.py into core/config.py --- code2DFD.py | 2 +- tmp/tmp.py => core/config.py | 0 core/dfd_extraction.py | 2 +- core/file_interaction.py | 2 +- core/parse_files.py | 2 +- core/technology_switch.py | 2 +- output_generators/codeable_model.py | 2 +- output_generators/codeable_models_to_plantuml.py | 2 +- output_generators/json_architecture.py | 2 +- output_generators/json_edges.py | 2 +- output_generators/plaintext.py | 2 +- output_generators/traceability.py | 2 +- .../database_connections/dbc_entry.py | 2 +- technology_specific_extractors/docker/dcr_entry.py | 2 +- technology_specific_extractors/docker_compose/dcm_entry.py | 2 +- technology_specific_extractors/docker_compose/dcm_parser.py | 2 +- technology_specific_extractors/feign_client/fgn_entry.py | 2 +- technology_specific_extractors/gradle/grd_entry.py | 2 +- technology_specific_extractors/html/html_entry.py | 2 +- .../implicit_connections/imp_entry.py | 2 +- technology_specific_extractors/kafka/kfk_entry.py | 2 +- technology_specific_extractors/maven/mvn_entry.py | 2 +- technology_specific_extractors/nginx/ngn_entry.py | 2 +- technology_specific_extractors/prometheus/prm_entry.py | 2 +- technology_specific_extractors/rabbitmq/rmq_entry.py | 2 +- technology_specific_extractors/resttemplate/rst_entry.py | 2 +- technology_specific_extractors/spring_config/cnf_entry.py | 2 +- technology_specific_extractors/turbine/trb_entry.py | 2 +- technology_specific_extractors/zuul/zul_entry.py | 2 +- 29 files changed, 28 insertions(+), 28 deletions(-) rename tmp/tmp.py => core/config.py (100%) diff --git a/code2DFD.py b/code2DFD.py index acc3f0e..b2e61c4 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -9,7 +9,7 @@ from core.dfd_extraction import perform_analysis from output_generators.logger import logger -import tmp.tmp as tmp +import core.config as tmp CONFIG_SECTIONS = ["Analysis Settings", "Repository"] DEFAULT_CONFIG = ConfigParser() diff --git a/tmp/tmp.py b/core/config.py similarity index 100% rename from tmp/tmp.py rename to core/config.py diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index 018cda8..3d1d8af 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -7,7 +7,7 @@ import output_generators.codeable_model as codeable_model import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.json_architecture as json_architecture import output_generators.json_edges as json_edges import output_generators.traceability as traceability diff --git a/core/file_interaction.py b/core/file_interaction.py index 4535eef..1f8167e 100644 --- a/core/file_interaction.py +++ b/core/file_interaction.py @@ -5,7 +5,7 @@ from output_generators.logger import logger import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp count = 0 diff --git a/core/parse_files.py b/core/parse_files.py index 8c04498..a01466c 100644 --- a/core/parse_files.py +++ b/core/parse_files.py @@ -4,7 +4,7 @@ import ruamel.yaml import core.file_interaction as fi -import tmp.tmp as tmp +import core.config as tmp from output_generators.logger import logger diff --git a/core/technology_switch.py b/core/technology_switch.py index e6f442b..0e3627c 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -11,7 +11,7 @@ import technology_specific_extractors.maven.mvn_entry as mvn import technology_specific_extractors.rabbitmq.rmq_entry as rmq import technology_specific_extractors.resttemplate.rst_entry as rst -import tmp.tmp as tmp +import core.config as tmp COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, diff --git a/output_generators/codeable_model.py b/output_generators/codeable_model.py index 716a62f..66a1a14 100644 --- a/output_generators/codeable_model.py +++ b/output_generators/codeable_model.py @@ -1,7 +1,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp # the used metamodel is microservice_dfds_metamodel.py diff --git a/output_generators/codeable_models_to_plantuml.py b/output_generators/codeable_models_to_plantuml.py index 47d4dc9..1316a74 100644 --- a/output_generators/codeable_models_to_plantuml.py +++ b/output_generators/codeable_models_to_plantuml.py @@ -1,7 +1,7 @@ import os.path from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp plantuml_new = str() diff --git a/output_generators/json_architecture.py b/output_generators/json_architecture.py index 2f40459..23da682 100644 --- a/output_generators/json_architecture.py +++ b/output_generators/json_architecture.py @@ -2,7 +2,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp def generate_json_architecture(microservices: dict, information_flows: dict, external_components: dict): diff --git a/output_generators/json_edges.py b/output_generators/json_edges.py index dcf5804..4439af7 100644 --- a/output_generators/json_edges.py +++ b/output_generators/json_edges.py @@ -7,7 +7,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp def generate_json_edges(information_flows: dict): diff --git a/output_generators/plaintext.py b/output_generators/plaintext.py index 614bf7d..6dc0751 100644 --- a/output_generators/plaintext.py +++ b/output_generators/plaintext.py @@ -1,7 +1,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp def write_plaintext(microservices, information_flows, external_components): diff --git a/output_generators/traceability.py b/output_generators/traceability.py index bb1330d..ca61cb1 100644 --- a/output_generators/traceability.py +++ b/output_generators/traceability.py @@ -2,7 +2,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +import core.config as tmp traceability = dict() traceability["nodes"] = dict() diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index e9b8bca..3945ee8 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -2,7 +2,7 @@ import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/docker/dcr_entry.py b/technology_specific_extractors/docker/dcr_entry.py index 45ea022..c151b40 100644 --- a/technology_specific_extractors/docker/dcr_entry.py +++ b/technology_specific_extractors/docker/dcr_entry.py @@ -1,6 +1,6 @@ import os -import tmp.tmp as tmp +import core.config as tmp def detect_port(path: str) -> int: diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index 37750ec..c3ca584 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -5,7 +5,7 @@ from output_generators.logger import logger import core.technology_switch as tech_sw import technology_specific_extractors.docker_compose.dcm_parser as dcm_parser -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability docker_compose_content = False diff --git a/technology_specific_extractors/docker_compose/dcm_parser.py b/technology_specific_extractors/docker_compose/dcm_parser.py index 7ec5e4b..7fc3a12 100644 --- a/technology_specific_extractors/docker_compose/dcm_parser.py +++ b/technology_specific_extractors/docker_compose/dcm_parser.py @@ -6,7 +6,7 @@ import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env -import tmp.tmp as tmp +import core.config as tmp # The following is taken from ruamel.yaml's authro as a workaround for getting line count for str objects diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index e6b0963..efe8475 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index f11b7d2..03be5fd 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -6,7 +6,7 @@ from output_generators.logger import logger import core.parse_files as parse import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index 418b981..f5e5fb9 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -3,7 +3,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw import output_generators.traceability as traceability -import tmp.tmp as tmp +import core.config as tmp def set_information_flows(dfd): diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index 8a89219..9944232 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -6,7 +6,7 @@ import core.file_interaction as fi from output_generators.logger import logger import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index 2575569..c3250ff 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -5,7 +5,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability from output_generators.logger import logger diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index 81c2d7a..80ef677 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -7,7 +7,7 @@ import core.parse_files as parse import core.technology_switch as tech_sw import technology_specific_extractors.docker.dcr_entry as dcr -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability try: diff --git a/technology_specific_extractors/nginx/ngn_entry.py b/technology_specific_extractors/nginx/ngn_entry.py index 30b5494..cc8814e 100644 --- a/technology_specific_extractors/nginx/ngn_entry.py +++ b/technology_specific_extractors/nginx/ngn_entry.py @@ -4,7 +4,7 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/prometheus/prm_entry.py b/technology_specific_extractors/prometheus/prm_entry.py index fcbf306..8add360 100644 --- a/technology_specific_extractors/prometheus/prm_entry.py +++ b/technology_specific_extractors/prometheus/prm_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index ab4d86b..49a2411 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -5,7 +5,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability from output_generators.logger import logger diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index 3e7fc65..ec04677 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -4,7 +4,7 @@ import core.technology_switch as tech_sw from output_generators.logger import logger import output_generators.traceability as traceability -import tmp.tmp as tmp +import core.config as tmp def set_information_flows(dfd) -> dict: diff --git a/technology_specific_extractors/spring_config/cnf_entry.py b/technology_specific_extractors/spring_config/cnf_entry.py index fa9508e..d739386 100644 --- a/technology_specific_extractors/spring_config/cnf_entry.py +++ b/technology_specific_extractors/spring_config/cnf_entry.py @@ -4,7 +4,7 @@ import core.parse_files as parse import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/turbine/trb_entry.py b/technology_specific_extractors/turbine/trb_entry.py index 6538dac..ee4e37a 100644 --- a/technology_specific_extractors/turbine/trb_entry.py +++ b/technology_specific_extractors/turbine/trb_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability diff --git a/technology_specific_extractors/zuul/zul_entry.py b/technology_specific_extractors/zuul/zul_entry.py index 6b4fcee..5d18a24 100644 --- a/technology_specific_extractors/zuul/zul_entry.py +++ b/technology_specific_extractors/zuul/zul_entry.py @@ -1,7 +1,7 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +import core.config as tmp import output_generators.traceability as traceability From 7fa5bc99a9b2539a67c6609698df0c53b5a1ff0f Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 20:20:25 +0200 Subject: [PATCH 05/10] refactor: temp_config into code2dfd_config --- code2DFD.py | 40 +++++++++---------- core/config.py | 2 +- core/dfd_extraction.py | 28 ++++++------- core/file_interaction.py | 14 +++---- core/parse_files.py | 2 +- core/technology_switch.py | 16 ++++---- output_generators/codeable_model.py | 4 +- .../codeable_models_to_plantuml.py | 2 +- output_generators/json_architecture.py | 2 +- output_generators/json_edges.py | 2 +- output_generators/plaintext.py | 2 +- output_generators/traceability.py | 2 +- .../database_connections/dbc_entry.py | 16 ++++---- .../docker/dcr_entry.py | 2 +- .../docker_compose/dcm_entry.py | 18 ++++----- .../docker_compose/dcm_parser.py | 6 +-- .../feign_client/fgn_entry.py | 6 +-- .../gradle/grd_entry.py | 10 ++--- .../html/html_entry.py | 6 +-- .../implicit_connections/imp_entry.py | 6 +-- .../kafka/kfk_entry.py | 10 ++--- .../maven/mvn_entry.py | 16 ++++---- .../nginx/ngn_entry.py | 2 +- .../prometheus/prm_entry.py | 2 +- .../rabbitmq/rmq_entry.py | 12 +++--- .../resttemplate/rst_entry.py | 10 ++--- .../spring_config/cnf_entry.py | 2 +- .../turbine/trb_entry.py | 2 +- .../zuul/zul_entry.py | 2 +- 29 files changed, 122 insertions(+), 122 deletions(-) diff --git a/code2DFD.py b/code2DFD.py index b2e61c4..8ab8eef 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -29,16 +29,16 @@ def api_invocation(url: str, commit: str) -> dict: logger.info("*** New execution ***") logger.debug("Initializing config to tmp file") for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.tmp_config.add_section(section) + tmp.code2dfd_config.add_section(section) for entry in DEFAULT_CONFIG[section]: - tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry]) + tmp.code2dfd_config.set(section, entry, DEFAULT_CONFIG[section][entry]) # Overwrite repo_path from config file with the one from the API call - tmp.tmp_config.set("Repository", "url", url) - tmp.tmp_config.set("Repository", "local_path", - os.path.join(os.getcwd(), "analysed_repositories")) + tmp.code2dfd_config.set("Repository", "url", url) + tmp.code2dfd_config.set("Repository", "local_path", + os.path.join(os.getcwd(), "analysed_repositories")) if commit is not None: - tmp.tmp_config.set("Repository", "commit", commit) + tmp.code2dfd_config.set("Repository", "commit", commit) # Call extraction codeable_models, traceability = perform_analysis() @@ -75,36 +75,36 @@ def cli_invocation(): if args.config_path: # Copy config to tmp file logger.debug("Copying config file to tmp file") - tmp.tmp_config.read(args.config_path) + tmp.code2dfd_config.read(args.config_path) else: # global ini_config logger.debug("Initializing tmp file with default config") for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.tmp_config.add_section(section) + tmp.code2dfd_config.add_section(section) for entry in DEFAULT_CONFIG[section]: - tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry]) + tmp.code2dfd_config.set(section, entry, DEFAULT_CONFIG[section][entry]) if args.repo_url: - tmp.tmp_config.set("Repository", "url", args.repo_url) + tmp.code2dfd_config.set("Repository", "url", args.repo_url) elif args.github_handle: - tmp.tmp_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") - elif not tmp.tmp_config.has_option("Repository", "url"): + tmp.code2dfd_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") + elif not tmp.code2dfd_config.has_option("Repository", "url"): raise AttributeError("Parameter [Repository][url] must be provided either in config file or by --repo_url") if args.repo_local_path: - tmp.tmp_config.set("Repository", "local_path", args.local_path) - elif not tmp.tmp_config.has_option("Repository", "local_path"): - tmp.tmp_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) + tmp.code2dfd_config.set("Repository", "local_path", args.local_path) + elif not tmp.code2dfd_config.has_option("Repository", "local_path"): + tmp.code2dfd_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) if args.development_mode: - tmp.tmp_config.set("Analysis Settings", "development_mode", "True") + tmp.code2dfd_config.set("Analysis Settings", "development_mode", "True") if args.commit is not None: commit = args.commit[:7] - tmp.tmp_config.set("Repository", "commit", commit) - elif tmp.tmp_config.has_option("Repository", "commit"): - commit = tmp.tmp_config.get("Repository", "commit")[:7] - tmp.tmp_config.set("Repository", "commit", commit) + tmp.code2dfd_config.set("Repository", "commit", commit) + elif tmp.code2dfd_config.has_option("Repository", "commit"): + commit = tmp.code2dfd_config.get("Repository", "commit")[:7] + tmp.code2dfd_config.set("Repository", "commit", commit) perform_analysis() diff --git a/core/config.py b/core/config.py index e07827c..2ca7708 100644 --- a/core/config.py +++ b/core/config.py @@ -1,3 +1,3 @@ from configparser import ConfigParser -tmp_config = ConfigParser() +code2dfd_config = ConfigParser() diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index 3d1d8af..c640031 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -55,21 +55,21 @@ def perform_analysis(): """ Entrypoint for the DFD extraction that initializes the repository """ - local_path = tmp.tmp_config.get("Repository", "local_path") - url_path = tmp.tmp_config.get("Repository", "url") - tmp.tmp_config.add_section("DFD") + local_path = tmp.code2dfd_config.get("Repository", "local_path") + url_path = tmp.code2dfd_config.get("Repository", "url") + tmp.code2dfd_config.add_section("DFD") os.makedirs(local_path, exist_ok=True) repository = Repository(path_to_repo=url_path, clone_repo_to=local_path) with repository._prep_repo(url_path) as git_repo: - tmp.tmp_config.set("Repository", "local_path", str(git_repo.path)) + tmp.code2dfd_config.set("Repository", "local_path", str(git_repo.path)) head = git_repo.get_head().hash[:7] - if tmp.tmp_config.has_option("Repository", "commit"): - commit = tmp.tmp_config.get("Repository", "commit") + if tmp.code2dfd_config.has_option("Repository", "commit"): + commit = tmp.code2dfd_config.get("Repository", "commit") else: commit = head repo_name = git_repo.project_name - tmp.tmp_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) + tmp.code2dfd_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) git_repo.checkout(commit) print(f"\nStart extraction of DFD for {repo_name} on commit {commit} at {datetime.now().strftime('%H:%M:%S')}") codeable_models, traceability_content = DFD_extraction() @@ -104,13 +104,13 @@ def DFD_extraction(): # Check authentication information of services microservices = detect_authentication_scopes(microservices, dfd) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Get information flows - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) new_information_flows = tech_sw.get_information_flows(dfd) - external_components = ast.literal_eval(tmp.tmp_config["DFD"]["external_components"]) + external_components = ast.literal_eval(tmp.code2dfd_config["DFD"]["external_components"]) # Merge old and new for new_flow in new_information_flows.keys(): @@ -147,9 +147,9 @@ def DFD_extraction(): print("\nFinished extraction") # Saving - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) plaintext.write_plaintext(microservices, information_flows, external_components) codeable_models, codeable_models_path = codeable_model.output_codeable_model(microservices, information_flows, external_components) @@ -171,7 +171,7 @@ def classify_brokers(microservices: dict) -> dict: microservices = detect_rabbitmq_server(microservices) microservices = detect_kafka_server(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices diff --git a/core/file_interaction.py b/core/file_interaction.py index 1f8167e..c6adcd0 100644 --- a/core/file_interaction.py +++ b/core/file_interaction.py @@ -61,7 +61,7 @@ def search_keywords(keywords: str): """Searches keywords locally using grep. """ - repo_folder = tmp.tmp_config["Repository"]["local_path"] + repo_folder = tmp.code2dfd_config["Repository"]["local_path"] results = dict() @@ -171,7 +171,7 @@ def file_as_lines(path): """Downloads and splits raw file into lines. """ - local_path = tmp.tmp_config.get("Repository", "local_path") + local_path = tmp.code2dfd_config.get("Repository", "local_path") local_path = os.path.join(local_path, path) with open(local_path, "r") as file: @@ -329,7 +329,7 @@ def resolve_url(url: str, microservice: str, dfd) -> str: def check_dockerfile(build_path: str): """Checks if under the service's build-path there is a dockerfile. If yes, returns it. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] # find docker-compose path, since build-path is relative to that raw_files = get_file_as_lines("docker-compose.yml") @@ -368,7 +368,7 @@ def file_exists(file_name: str) -> bool: """Checks if a file exists in the repository. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(local_repo_path)) @@ -391,7 +391,7 @@ def get_repo_contents_local(path: str) -> set: repo = set() - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] to_crawl = local_repo_path if path: @@ -414,7 +414,7 @@ def get_file_as_yaml(filename: str) -> dict: files = dict() - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = tmp.code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() @@ -437,7 +437,7 @@ def get_file_as_lines(filename: str) -> dict: files = dict() - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = tmp.code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() diff --git a/core/parse_files.py b/core/parse_files.py index a01466c..e3b294d 100644 --- a/core/parse_files.py +++ b/core/parse_files.py @@ -173,7 +173,7 @@ def parse_yaml_file(file_path: str) -> str: """ - local_path = tmp.tmp_config.get("Repository", "local_path") + local_path = tmp.code2dfd_config.get("Repository", "local_path") file_path = os.path.join(local_path, file_path) yaml = ruamel.yaml.YAML() yaml.Constructor = MyConstructor diff --git a/core/technology_switch.py b/core/technology_switch.py index 0e3627c..a31e730 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -22,31 +22,31 @@ def get_microservices(dfd) -> dict: """Calls get_microservices from correct container technology or returns existing list. """ - if tmp.tmp_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + return ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) else: logger.info("Microservices not set yet, start extraction") mvn.set_microservices(dfd) grd.set_microservices(dfd) dcm.set_microservices(dfd) - if tmp.tmp_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + return ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) def get_information_flows(dfd) -> dict: """Calls get_information_flows from correct communication technology. """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + return ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: logger.info("Information flows not set yet, start extraction") for func in COMMUNICATIONS_TECH_LIST.values(): func.set_information_flows(dfd) - if tmp.tmp_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + return ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) def detect_microservice(file_path: str, dfd) -> str: diff --git a/output_generators/codeable_model.py b/output_generators/codeable_model.py index 66a1a14..0eaedd7 100644 --- a/output_generators/codeable_model.py +++ b/output_generators/codeable_model.py @@ -9,7 +9,7 @@ def output_codeable_model(microservices, information_flows, external_components): """Entry function to creation of codeable models. Calls all necessary helper functions and outputs the codeable model""" - parts = Path(tmp.tmp_config["Analysis Settings"]["output_path"]).parts + parts = Path(tmp.code2dfd_config["Analysis Settings"]["output_path"]).parts model_name = f"{parts[-2]}_{parts[-1]}" file_content = header() @@ -129,7 +129,7 @@ def create_file(model_name: str, content: str): """Writes content to file. """ model_name = model_name.replace("-", "_") - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] filename = f"{model_name}.py" output_path = os.path.join(output_path, filename) diff --git a/output_generators/codeable_models_to_plantuml.py b/output_generators/codeable_models_to_plantuml.py index 1316a74..904d888 100644 --- a/output_generators/codeable_models_to_plantuml.py +++ b/output_generators/codeable_models_to_plantuml.py @@ -13,7 +13,7 @@ def convert(codeable_models_path: str) -> str: global plantuml_new - output_file_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_file_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_file_path).parts filename = f"{parts[-2]}--{parts[-1]}_uml.txt" output_file_path = os.path.join(output_file_path, filename) diff --git a/output_generators/json_architecture.py b/output_generators/json_architecture.py index 23da682..69933c8 100644 --- a/output_generators/json_architecture.py +++ b/output_generators/json_architecture.py @@ -13,7 +13,7 @@ def generate_json_architecture(microservices: dict, information_flows: dict, ext "information_flows": list(information_flows.values()), "external_components": list(external_components.values())} - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_json_architecture.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/json_edges.py b/output_generators/json_edges.py index 4439af7..5c32de1 100644 --- a/output_generators/json_edges.py +++ b/output_generators/json_edges.py @@ -33,7 +33,7 @@ def write_to_file(architecture_dict): """Writes json architecture to file. """ - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_edges.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/plaintext.py b/output_generators/plaintext.py index 6dc0751..bdfae3a 100644 --- a/output_generators/plaintext.py +++ b/output_generators/plaintext.py @@ -5,7 +5,7 @@ def write_plaintext(microservices, information_flows, external_components): - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_results.txt" output_path = os.path.join(output_path, filename) diff --git a/output_generators/traceability.py b/output_generators/traceability.py index ca61cb1..9a05103 100644 --- a/output_generators/traceability.py +++ b/output_generators/traceability.py @@ -154,7 +154,7 @@ def write_to_file(): """Writes tracebility info from dict to json file. """ - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_traceability.json" output_path = os.path.join(output_path, filename) diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index 3945ee8..c7e713c 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -10,13 +10,13 @@ def set_information_flows(dfd) -> set: """Goes through services and checks if there are connections to databases. """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() - if tmp.tmp_config.has_option("DFD", "external_components"): - external_components = ast.literal_eval(tmp.tmp_config["DFD"]["external_components"]) + if tmp.code2dfd_config.has_option("DFD", "external_components"): + external_components = ast.literal_eval(tmp.code2dfd_config["DFD"]["external_components"]) else: external_components = dict() @@ -24,9 +24,9 @@ def set_information_flows(dfd) -> set: microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) return microservices, information_flows @@ -220,7 +220,7 @@ def check_properties(microservices: dict, information_flows: dict, external_comp except: information_flows[id]["tagged_values"] = [("Username", username.strip())] - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) trace = dict() trace["item"] = "database-" + str(microservices[m]["name"]) + " -> " + microservices[m]["name"] diff --git a/technology_specific_extractors/docker/dcr_entry.py b/technology_specific_extractors/docker/dcr_entry.py index c151b40..b5ac0d6 100644 --- a/technology_specific_extractors/docker/dcr_entry.py +++ b/technology_specific_extractors/docker/dcr_entry.py @@ -8,7 +8,7 @@ def detect_port(path: str) -> int: """ port = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, os.path.dirname(path)))) diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index c3ca584..16e1549 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -29,7 +29,7 @@ def set_microservices(dfd) -> None: if len(raw_files) == 0: microservices = tech_sw.get_microservices(dfd) microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return docker_compose_content = raw_files[0]["content"] @@ -38,12 +38,12 @@ def set_microservices(dfd) -> None: if not microservices_set: microservices = tech_sw.get_microservices(dfd) microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices = dictionarify(microservices_set, properties_dict) microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) def clean_pom_names(microservices: dict) -> dict: @@ -60,8 +60,8 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: """Turns set of services into dictionary. """ - if tmp.tmp_config.has_option("DFD", "microservices"): - elements = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + elements = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) else: elements = dict() @@ -115,8 +115,8 @@ def set_information_flows(dfd): global docker_compose_content - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -135,7 +135,7 @@ def set_information_flows(dfd): information_flows = dcm_parser.extract_information_flows(docker_compose_content, microservices, information_flows) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -176,7 +176,7 @@ def detect_microservice(file_path: str, dfd) -> str: microservice = False dockerfile_path = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] # Find corresponding dockerfile dirs = list() diff --git a/technology_specific_extractors/docker_compose/dcm_parser.py b/technology_specific_extractors/docker_compose/dcm_parser.py index 7fc3a12..81dd8b5 100644 --- a/technology_specific_extractors/docker_compose/dcm_parser.py +++ b/technology_specific_extractors/docker_compose/dcm_parser.py @@ -67,8 +67,8 @@ def extract_microservices(file_content, file_name) -> set: image = False build = False - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices_dict = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + microservices_dict = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) else: microservices_dict= dict() microservices_set = set() @@ -621,7 +621,7 @@ def extract_microservices(file_content, file_name) -> set: else: microservices_dict[correct_id]["properties"] = properties - tmp.tmp_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) return microservices_set, properties_dict diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index efe8475..b754624 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -12,8 +12,8 @@ def set_information_flows(dfd) -> dict: microservices = tech_sw.get_microservices(dfd) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -94,7 +94,7 @@ def set_information_flows(dfd) -> dict: traceability.add_trace(trace) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index 03be5fd..574c712 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -17,8 +17,8 @@ def set_microservices(dfd) -> dict: if not used_in_application(): return False - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + microservices = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) else: microservices = dict() @@ -56,7 +56,7 @@ def set_microservices(dfd) -> dict: except: pass - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices @@ -89,7 +89,7 @@ def parse_properties_file(gradle_path: str): # find properties file path = os.path.dirname(gradle_path) - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -135,7 +135,7 @@ def detect_microservice(file_path, dfd): found_gradle = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(file_path) diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index f5e5fb9..c871e42 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -12,8 +12,8 @@ def set_information_flows(dfd): microservices = tech_sw.get_microservices(dfd) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -52,5 +52,5 @@ def set_information_flows(dfd): except: pass - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index 9944232..6cc3ae3 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -14,8 +14,8 @@ def set_information_flows(dfd) -> dict: """Adds connections based on parsed config files. """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -40,7 +40,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index c3250ff..7b68e6c 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -16,8 +16,8 @@ def set_information_flows(dfd) -> set: """Connects incoming endpoints, outgoing endpoints, and routings to information flows """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -39,7 +39,7 @@ def set_information_flows(dfd) -> set: information_flows = detect_stream_binders(microservices, information_flows, dfd) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -184,8 +184,8 @@ def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints # incoming: (topic, microservice, (file, line, span)) # outgoing: (topic, microservice, asset, (file, line, span)) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index 80ef677..87aa583 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -23,8 +23,8 @@ def set_microservices(dfd) -> dict: """Extracts the list of services from pom.xml files and sets the variable in the tmp-file. """ - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + if tmp.code2dfd_config.has_option("DFD", "microservices"): + microservices = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) else: microservices = dict() microservices_set = set() @@ -71,7 +71,7 @@ def set_microservices(dfd) -> dict: nested_microservices = check_nested_modules(module_dict) microservices_set.update(nested_microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser return microservices @@ -81,7 +81,7 @@ def extract_dependencies(properties: set, pom_file) -> set: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -100,7 +100,7 @@ def extract_modules(pom_file: dict) -> list: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -146,7 +146,7 @@ def parse_properties_file(pom_path: str): # find properties file path = os.path.dirname(pom_path) - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -185,7 +185,7 @@ def extract_servicename_pom_file(pom_file) -> str: microservice = [False, False] file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -224,7 +224,7 @@ def detect_microservice(file_path, dfd): path = file_path found_pom = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(path) diff --git a/technology_specific_extractors/nginx/ngn_entry.py b/technology_specific_extractors/nginx/ngn_entry.py index cc8814e..9104698 100644 --- a/technology_specific_extractors/nginx/ngn_entry.py +++ b/technology_specific_extractors/nginx/ngn_entry.py @@ -101,7 +101,7 @@ def detect_nginx(microservices: dict, information_flows: dict, external_componen id = 0 microservices[id] = dict() - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] docker_path = os.path.dirname(results[r]["path"]) if docker_path and local_repo_path: docker_path = os.path.relpath(docker_path, start=local_repo_path) diff --git a/technology_specific_extractors/prometheus/prm_entry.py b/technology_specific_extractors/prometheus/prm_entry.py index 8add360..cb3ec24 100644 --- a/technology_specific_extractors/prometheus/prm_entry.py +++ b/technology_specific_extractors/prometheus/prm_entry.py @@ -59,7 +59,7 @@ def detect_connections(microservices: dict, information_flows: dict, dockerfile, """Parses config file to find connections to prometheus. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] for line in dockerfile["content"]: if "ADD" in line: diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index 49a2411..f2e1340 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -17,8 +17,8 @@ def set_information_flows(dfd) -> set: if not used_in_application(): return - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -37,7 +37,7 @@ def set_information_flows(dfd) -> set: id = 0 information_flows[id] = new_information_flows[ni] - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -155,8 +155,8 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo # outgoing: (exchange, routingkey, microservice, (file, line, span)) # incoming: (queue, microservice, (file, line, span)) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -304,7 +304,7 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo traceability.add_trace(trace) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index ec04677..c6303a1 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -14,8 +14,8 @@ def set_information_flows(dfd) -> dict: if not used_in_application(): return - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + if tmp.code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -31,7 +31,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -91,7 +91,7 @@ def get_incoming_endpoints(dfd) -> list: bracket_count = adjust_bracket_count(bracket_count, line) - tmp.tmp_config.set("DFD", "endpoints", str(endpoints)) + tmp.code2dfd_config.set("DFD", "endpoints", str(endpoints)) return endpoints @@ -135,7 +135,7 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): microservices[m]["tagged_values"].append(('Endpoints', list(ordered_endpoints[endpoint]))) else: microservices[m]["tagged_values"] = [('Endpoints', list(ordered_endpoints[endpoint]))] - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) def get_outgoing_endpoints(information_flows: dict, dfd) -> set: diff --git a/technology_specific_extractors/spring_config/cnf_entry.py b/technology_specific_extractors/spring_config/cnf_entry.py index d739386..f2bc4b4 100644 --- a/technology_specific_extractors/spring_config/cnf_entry.py +++ b/technology_specific_extractors/spring_config/cnf_entry.py @@ -229,7 +229,7 @@ def parse_config_files(config_server: str, config_file_path: str, config_file_pa # external (other github repository) didn't work, look locally if config_file_path_local: - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = tmp.code2dfd_config["Repository"]["local_path"] config_file_path_local = os.path.relpath(config_file_path_local, start=local_path) new_contents = fi.get_repo_contents_local(config_file_path_local) diff --git a/technology_specific_extractors/turbine/trb_entry.py b/technology_specific_extractors/turbine/trb_entry.py index ee4e37a..fe233bd 100644 --- a/technology_specific_extractors/turbine/trb_entry.py +++ b/technology_specific_extractors/turbine/trb_entry.py @@ -155,7 +155,7 @@ def detect_turbine_stream(microservices: dict, information_flows: dict, dfd) -> found_pom = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] dirs = list() diff --git a/technology_specific_extractors/zuul/zul_entry.py b/technology_specific_extractors/zuul/zul_entry.py index 5d18a24..ef5ff3a 100644 --- a/technology_specific_extractors/zuul/zul_entry.py +++ b/technology_specific_extractors/zuul/zul_entry.py @@ -112,5 +112,5 @@ def detect_zuul(microservices: dict, information_flows: dict, external_component except: information_flows[id]["tagged_values"] = [("Load Balancer", load_balancer)] - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) return microservices, information_flows, external_components From a849fcf19987159c7b81e0467ee93fc139eec6b8 Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 20:24:36 +0200 Subject: [PATCH 06/10] refactor: config defaults into config.py --- code2DFD.py | 21 ++------------------- core/config.py | 5 +++++ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/code2DFD.py b/code2DFD.py index 8ab8eef..f7fd7e6 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -3,7 +3,6 @@ # Author: Simon Schneider, 2023 # Contact: simon.schneider@tuhh.de import os -from configparser import ConfigParser from datetime import datetime import argparse @@ -11,12 +10,6 @@ from output_generators.logger import logger import core.config as tmp -CONFIG_SECTIONS = ["Analysis Settings", "Repository"] -DEFAULT_CONFIG = ConfigParser() -for section in CONFIG_SECTIONS: - DEFAULT_CONFIG.add_section(section) -DEFAULT_CONFIG.set("Analysis Settings", "development_mode", "False") - def api_invocation(url: str, commit: str) -> dict: """Entry function for when tool is called via API call. @@ -27,11 +20,7 @@ def api_invocation(url: str, commit: str) -> dict: start_time = datetime.now() logger.info("*** New execution ***") - logger.debug("Initializing config to tmp file") - for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.code2dfd_config.add_section(section) - for entry in DEFAULT_CONFIG[section]: - tmp.code2dfd_config.set(section, entry, DEFAULT_CONFIG[section][entry]) + logger.debug("Initializing config") # Overwrite repo_path from config file with the one from the API call tmp.code2dfd_config.set("Repository", "url", url) @@ -74,15 +63,9 @@ def cli_invocation(): if args.config_path: # Copy config to tmp file - logger.debug("Copying config file to tmp file") + logger.debug("Copying config file to internal config") tmp.code2dfd_config.read(args.config_path) - else: # global ini_config - logger.debug("Initializing tmp file with default config") - for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.code2dfd_config.add_section(section) - for entry in DEFAULT_CONFIG[section]: - tmp.code2dfd_config.set(section, entry, DEFAULT_CONFIG[section][entry]) if args.repo_url: tmp.code2dfd_config.set("Repository", "url", args.repo_url) diff --git a/core/config.py b/core/config.py index 2ca7708..2b709de 100644 --- a/core/config.py +++ b/core/config.py @@ -1,3 +1,8 @@ from configparser import ConfigParser code2dfd_config = ConfigParser() + +CONFIG_SECTIONS = ["Analysis Settings", "Repository"] +for section in CONFIG_SECTIONS: + code2dfd_config.add_section(section) +code2dfd_config.set("Analysis Settings", "development_mode", "False") From c6e39f639977442556ebe629289797edfc359fa0 Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 20:48:16 +0200 Subject: [PATCH 07/10] refactor: import code2dfd_config in a more readable way --- code2DFD.py | 32 +++++++++---------- core/dfd_extraction.py | 30 ++++++++--------- core/file_interaction.py | 16 +++++----- core/parse_files.py | 4 +-- core/technology_switch.py | 18 +++++------ output_generators/codeable_model.py | 6 ++-- .../codeable_models_to_plantuml.py | 4 +-- output_generators/json_architecture.py | 4 +-- output_generators/json_edges.py | 4 +-- output_generators/plaintext.py | 4 +-- output_generators/traceability.py | 4 +-- .../database_connections/dbc_entry.py | 18 +++++------ .../docker/dcr_entry.py | 4 +-- .../docker_compose/dcm_entry.py | 20 ++++++------ .../docker_compose/dcm_parser.py | 8 ++--- .../feign_client/fgn_entry.py | 8 ++--- .../gradle/grd_entry.py | 12 +++---- .../html/html_entry.py | 8 ++--- .../implicit_connections/imp_entry.py | 8 ++--- .../kafka/kfk_entry.py | 12 +++---- .../maven/mvn_entry.py | 18 +++++------ .../nginx/ngn_entry.py | 4 +-- .../prometheus/prm_entry.py | 4 +-- .../rabbitmq/rmq_entry.py | 14 ++++---- .../resttemplate/rst_entry.py | 12 +++---- .../spring_config/cnf_entry.py | 4 +-- .../turbine/trb_entry.py | 4 +-- 27 files changed, 142 insertions(+), 142 deletions(-) diff --git a/code2DFD.py b/code2DFD.py index f7fd7e6..23486da 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -8,7 +8,7 @@ from core.dfd_extraction import perform_analysis from output_generators.logger import logger -import core.config as tmp +from core.config import code2dfd_config def api_invocation(url: str, commit: str) -> dict: @@ -23,11 +23,11 @@ def api_invocation(url: str, commit: str) -> dict: logger.debug("Initializing config") # Overwrite repo_path from config file with the one from the API call - tmp.code2dfd_config.set("Repository", "url", url) - tmp.code2dfd_config.set("Repository", "local_path", + code2dfd_config.set("Repository", "url", url) + code2dfd_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) if commit is not None: - tmp.code2dfd_config.set("Repository", "commit", commit) + code2dfd_config.set("Repository", "commit", commit) # Call extraction codeable_models, traceability = perform_analysis() @@ -64,30 +64,30 @@ def cli_invocation(): if args.config_path: # Copy config to tmp file logger.debug("Copying config file to internal config") - tmp.code2dfd_config.read(args.config_path) + code2dfd_config.read(args.config_path) # global ini_config if args.repo_url: - tmp.code2dfd_config.set("Repository", "url", args.repo_url) + code2dfd_config.set("Repository", "url", args.repo_url) elif args.github_handle: - tmp.code2dfd_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") - elif not tmp.code2dfd_config.has_option("Repository", "url"): + code2dfd_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") + elif not code2dfd_config.has_option("Repository", "url"): raise AttributeError("Parameter [Repository][url] must be provided either in config file or by --repo_url") if args.repo_local_path: - tmp.code2dfd_config.set("Repository", "local_path", args.local_path) - elif not tmp.code2dfd_config.has_option("Repository", "local_path"): - tmp.code2dfd_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) + code2dfd_config.set("Repository", "local_path", args.local_path) + elif not code2dfd_config.has_option("Repository", "local_path"): + code2dfd_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) if args.development_mode: - tmp.code2dfd_config.set("Analysis Settings", "development_mode", "True") + code2dfd_config.set("Analysis Settings", "development_mode", "True") if args.commit is not None: commit = args.commit[:7] - tmp.code2dfd_config.set("Repository", "commit", commit) - elif tmp.code2dfd_config.has_option("Repository", "commit"): - commit = tmp.code2dfd_config.get("Repository", "commit")[:7] - tmp.code2dfd_config.set("Repository", "commit", commit) + code2dfd_config.set("Repository", "commit", commit) + elif code2dfd_config.has_option("Repository", "commit"): + commit = code2dfd_config.get("Repository", "commit")[:7] + code2dfd_config.set("Repository", "commit", commit) perform_analysis() diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index c640031..cf9143e 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -7,7 +7,7 @@ import output_generators.codeable_model as codeable_model import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.json_architecture as json_architecture import output_generators.json_edges as json_edges import output_generators.traceability as traceability @@ -55,21 +55,21 @@ def perform_analysis(): """ Entrypoint for the DFD extraction that initializes the repository """ - local_path = tmp.code2dfd_config.get("Repository", "local_path") - url_path = tmp.code2dfd_config.get("Repository", "url") - tmp.code2dfd_config.add_section("DFD") + local_path = code2dfd_config.get("Repository", "local_path") + url_path = code2dfd_config.get("Repository", "url") + code2dfd_config.add_section("DFD") os.makedirs(local_path, exist_ok=True) repository = Repository(path_to_repo=url_path, clone_repo_to=local_path) with repository._prep_repo(url_path) as git_repo: - tmp.code2dfd_config.set("Repository", "local_path", str(git_repo.path)) + code2dfd_config.set("Repository", "local_path", str(git_repo.path)) head = git_repo.get_head().hash[:7] - if tmp.code2dfd_config.has_option("Repository", "commit"): - commit = tmp.code2dfd_config.get("Repository", "commit") + if code2dfd_config.has_option("Repository", "commit"): + commit = code2dfd_config.get("Repository", "commit") else: commit = head repo_name = git_repo.project_name - tmp.code2dfd_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) + code2dfd_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) git_repo.checkout(commit) print(f"\nStart extraction of DFD for {repo_name} on commit {commit} at {datetime.now().strftime('%H:%M:%S')}") codeable_models, traceability_content = DFD_extraction() @@ -104,13 +104,13 @@ def DFD_extraction(): # Check authentication information of services microservices = detect_authentication_scopes(microservices, dfd) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Get information flows - tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) new_information_flows = tech_sw.get_information_flows(dfd) - external_components = ast.literal_eval(tmp.code2dfd_config["DFD"]["external_components"]) + external_components = ast.literal_eval(code2dfd_config["DFD"]["external_components"]) # Merge old and new for new_flow in new_information_flows.keys(): @@ -147,9 +147,9 @@ def DFD_extraction(): print("\nFinished extraction") # Saving - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) plaintext.write_plaintext(microservices, information_flows, external_components) codeable_models, codeable_models_path = codeable_model.output_codeable_model(microservices, information_flows, external_components) @@ -171,7 +171,7 @@ def classify_brokers(microservices: dict) -> dict: microservices = detect_rabbitmq_server(microservices) microservices = detect_kafka_server(microservices) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices diff --git a/core/file_interaction.py b/core/file_interaction.py index c6adcd0..6923fff 100644 --- a/core/file_interaction.py +++ b/core/file_interaction.py @@ -5,7 +5,7 @@ from output_generators.logger import logger import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config count = 0 @@ -61,7 +61,7 @@ def search_keywords(keywords: str): """Searches keywords locally using grep. """ - repo_folder = tmp.code2dfd_config["Repository"]["local_path"] + repo_folder = code2dfd_config["Repository"]["local_path"] results = dict() @@ -171,7 +171,7 @@ def file_as_lines(path): """Downloads and splits raw file into lines. """ - local_path = tmp.code2dfd_config.get("Repository", "local_path") + local_path = code2dfd_config.get("Repository", "local_path") local_path = os.path.join(local_path, path) with open(local_path, "r") as file: @@ -329,7 +329,7 @@ def resolve_url(url: str, microservice: str, dfd) -> str: def check_dockerfile(build_path: str): """Checks if under the service's build-path there is a dockerfile. If yes, returns it. """ - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] # find docker-compose path, since build-path is relative to that raw_files = get_file_as_lines("docker-compose.yml") @@ -368,7 +368,7 @@ def file_exists(file_name: str) -> bool: """Checks if a file exists in the repository. """ - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(local_repo_path)) @@ -391,7 +391,7 @@ def get_repo_contents_local(path: str) -> set: repo = set() - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] to_crawl = local_repo_path if path: @@ -414,7 +414,7 @@ def get_file_as_yaml(filename: str) -> dict: files = dict() - local_path = tmp.code2dfd_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() @@ -437,7 +437,7 @@ def get_file_as_lines(filename: str) -> dict: files = dict() - local_path = tmp.code2dfd_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() diff --git a/core/parse_files.py b/core/parse_files.py index e3b294d..7334378 100644 --- a/core/parse_files.py +++ b/core/parse_files.py @@ -4,7 +4,7 @@ import ruamel.yaml import core.file_interaction as fi -import core.config as tmp +from core.config import code2dfd_config from output_generators.logger import logger @@ -173,7 +173,7 @@ def parse_yaml_file(file_path: str) -> str: """ - local_path = tmp.code2dfd_config.get("Repository", "local_path") + local_path = code2dfd_config.get("Repository", "local_path") file_path = os.path.join(local_path, file_path) yaml = ruamel.yaml.YAML() yaml.Constructor = MyConstructor diff --git a/core/technology_switch.py b/core/technology_switch.py index a31e730..af9c642 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -11,7 +11,7 @@ import technology_specific_extractors.maven.mvn_entry as mvn import technology_specific_extractors.rabbitmq.rmq_entry as rmq import technology_specific_extractors.resttemplate.rst_entry as rst -import core.config as tmp +from core.config import code2dfd_config COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, @@ -22,31 +22,31 @@ def get_microservices(dfd) -> dict: """Calls get_microservices from correct container technology or returns existing list. """ - if tmp.code2dfd_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + return ast.literal_eval(code2dfd_config["DFD"]["microservices"]) else: logger.info("Microservices not set yet, start extraction") mvn.set_microservices(dfd) grd.set_microservices(dfd) dcm.set_microservices(dfd) - if tmp.code2dfd_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + return ast.literal_eval(code2dfd_config["DFD"]["microservices"]) def get_information_flows(dfd) -> dict: """Calls get_information_flows from correct communication technology. """ - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + return ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: logger.info("Information flows not set yet, start extraction") for func in COMMUNICATIONS_TECH_LIST.values(): func.set_information_flows(dfd) - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + return ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) def detect_microservice(file_path: str, dfd) -> str: diff --git a/output_generators/codeable_model.py b/output_generators/codeable_model.py index 0eaedd7..9ad3139 100644 --- a/output_generators/codeable_model.py +++ b/output_generators/codeable_model.py @@ -1,7 +1,7 @@ import os from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config # the used metamodel is microservice_dfds_metamodel.py @@ -9,7 +9,7 @@ def output_codeable_model(microservices, information_flows, external_components): """Entry function to creation of codeable models. Calls all necessary helper functions and outputs the codeable model""" - parts = Path(tmp.code2dfd_config["Analysis Settings"]["output_path"]).parts + parts = Path(code2dfd_config["Analysis Settings"]["output_path"]).parts model_name = f"{parts[-2]}_{parts[-1]}" file_content = header() @@ -129,7 +129,7 @@ def create_file(model_name: str, content: str): """Writes content to file. """ model_name = model_name.replace("-", "_") - output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] filename = f"{model_name}.py" output_path = os.path.join(output_path, filename) diff --git a/output_generators/codeable_models_to_plantuml.py b/output_generators/codeable_models_to_plantuml.py index 904d888..a65d991 100644 --- a/output_generators/codeable_models_to_plantuml.py +++ b/output_generators/codeable_models_to_plantuml.py @@ -1,7 +1,7 @@ import os.path from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config plantuml_new = str() @@ -13,7 +13,7 @@ def convert(codeable_models_path: str) -> str: global plantuml_new - output_file_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_file_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_file_path).parts filename = f"{parts[-2]}--{parts[-1]}_uml.txt" output_file_path = os.path.join(output_file_path, filename) diff --git a/output_generators/json_architecture.py b/output_generators/json_architecture.py index 69933c8..9b7ab15 100644 --- a/output_generators/json_architecture.py +++ b/output_generators/json_architecture.py @@ -2,7 +2,7 @@ import os from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config def generate_json_architecture(microservices: dict, information_flows: dict, external_components: dict): @@ -13,7 +13,7 @@ def generate_json_architecture(microservices: dict, information_flows: dict, ext "information_flows": list(information_flows.values()), "external_components": list(external_components.values())} - output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_json_architecture.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/json_edges.py b/output_generators/json_edges.py index 5c32de1..b3b74e6 100644 --- a/output_generators/json_edges.py +++ b/output_generators/json_edges.py @@ -7,7 +7,7 @@ import os from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config def generate_json_edges(information_flows: dict): @@ -33,7 +33,7 @@ def write_to_file(architecture_dict): """Writes json architecture to file. """ - output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_edges.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/plaintext.py b/output_generators/plaintext.py index bdfae3a..a999970 100644 --- a/output_generators/plaintext.py +++ b/output_generators/plaintext.py @@ -1,11 +1,11 @@ import os from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config def write_plaintext(microservices, information_flows, external_components): - output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_results.txt" output_path = os.path.join(output_path, filename) diff --git a/output_generators/traceability.py b/output_generators/traceability.py index 9a05103..5c439a9 100644 --- a/output_generators/traceability.py +++ b/output_generators/traceability.py @@ -2,7 +2,7 @@ import os from pathlib import Path -import core.config as tmp +from core.config import code2dfd_config traceability = dict() traceability["nodes"] = dict() @@ -154,7 +154,7 @@ def write_to_file(): """Writes tracebility info from dict to json file. """ - output_path = tmp.code2dfd_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_traceability.json" output_path = os.path.join(output_path, filename) diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index c7e713c..40a9b31 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -2,7 +2,7 @@ import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -10,13 +10,13 @@ def set_information_flows(dfd) -> set: """Goes through services and checks if there are connections to databases. """ - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() - if tmp.code2dfd_config.has_option("DFD", "external_components"): - external_components = ast.literal_eval(tmp.code2dfd_config["DFD"]["external_components"]) + if code2dfd_config.has_option("DFD", "external_components"): + external_components = ast.literal_eval(code2dfd_config["DFD"]["external_components"]) else: external_components = dict() @@ -24,9 +24,9 @@ def set_information_flows(dfd) -> set: microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) return microservices, information_flows @@ -220,7 +220,7 @@ def check_properties(microservices: dict, information_flows: dict, external_comp except: information_flows[id]["tagged_values"] = [("Username", username.strip())] - tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) + code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) trace = dict() trace["item"] = "database-" + str(microservices[m]["name"]) + " -> " + microservices[m]["name"] diff --git a/technology_specific_extractors/docker/dcr_entry.py b/technology_specific_extractors/docker/dcr_entry.py index b5ac0d6..ff5ab5e 100644 --- a/technology_specific_extractors/docker/dcr_entry.py +++ b/technology_specific_extractors/docker/dcr_entry.py @@ -1,6 +1,6 @@ import os -import core.config as tmp +from core.config import code2dfd_config def detect_port(path: str) -> int: @@ -8,7 +8,7 @@ def detect_port(path: str) -> int: """ port = False - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, os.path.dirname(path)))) diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index 16e1549..ad1fbbc 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -5,7 +5,7 @@ from output_generators.logger import logger import core.technology_switch as tech_sw import technology_specific_extractors.docker_compose.dcm_parser as dcm_parser -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability docker_compose_content = False @@ -29,7 +29,7 @@ def set_microservices(dfd) -> None: if len(raw_files) == 0: microservices = tech_sw.get_microservices(dfd) microservices = clean_pom_names(microservices) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return docker_compose_content = raw_files[0]["content"] @@ -38,12 +38,12 @@ def set_microservices(dfd) -> None: if not microservices_set: microservices = tech_sw.get_microservices(dfd) microservices = clean_pom_names(microservices) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices = dictionarify(microservices_set, properties_dict) microservices = clean_pom_names(microservices) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) def clean_pom_names(microservices: dict) -> dict: @@ -60,8 +60,8 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: """Turns set of services into dictionary. """ - if tmp.code2dfd_config.has_option("DFD", "microservices"): - elements = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + elements = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) else: elements = dict() @@ -115,8 +115,8 @@ def set_information_flows(dfd): global docker_compose_content - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -135,7 +135,7 @@ def set_information_flows(dfd): information_flows = dcm_parser.extract_information_flows(docker_compose_content, microservices, information_flows) - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -176,7 +176,7 @@ def detect_microservice(file_path: str, dfd) -> str: microservice = False dockerfile_path = False - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] # Find corresponding dockerfile dirs = list() diff --git a/technology_specific_extractors/docker_compose/dcm_parser.py b/technology_specific_extractors/docker_compose/dcm_parser.py index 81dd8b5..14c5b8a 100644 --- a/technology_specific_extractors/docker_compose/dcm_parser.py +++ b/technology_specific_extractors/docker_compose/dcm_parser.py @@ -6,7 +6,7 @@ import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env -import core.config as tmp +from core.config import code2dfd_config # The following is taken from ruamel.yaml's authro as a workaround for getting line count for str objects @@ -67,8 +67,8 @@ def extract_microservices(file_content, file_name) -> set: image = False build = False - if tmp.code2dfd_config.has_option("DFD", "microservices"): - microservices_dict = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + microservices_dict = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) else: microservices_dict= dict() microservices_set = set() @@ -621,7 +621,7 @@ def extract_microservices(file_content, file_name) -> set: else: microservices_dict[correct_id]["properties"] = properties - tmp.code2dfd_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) return microservices_set, properties_dict diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index b754624..b4ea4df 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -12,8 +12,8 @@ def set_information_flows(dfd) -> dict: microservices = tech_sw.get_microservices(dfd) - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -94,7 +94,7 @@ def set_information_flows(dfd) -> dict: traceability.add_trace(trace) - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index 574c712..0a62032 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -6,7 +6,7 @@ from output_generators.logger import logger import core.parse_files as parse import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -17,8 +17,8 @@ def set_microservices(dfd) -> dict: if not used_in_application(): return False - if tmp.code2dfd_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + microservices = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) else: microservices = dict() @@ -56,7 +56,7 @@ def set_microservices(dfd) -> dict: except: pass - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return microservices @@ -89,7 +89,7 @@ def parse_properties_file(gradle_path: str): # find properties file path = os.path.dirname(gradle_path) - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -135,7 +135,7 @@ def detect_microservice(file_path, dfd): found_gradle = False - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(file_path) diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index c871e42..15f6fbe 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -3,7 +3,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw import output_generators.traceability as traceability -import core.config as tmp +from core.config import code2dfd_config def set_information_flows(dfd): @@ -12,8 +12,8 @@ def set_information_flows(dfd): microservices = tech_sw.get_microservices(dfd) - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -52,5 +52,5 @@ def set_information_flows(dfd): except: pass - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index 6cc3ae3..b51fb9e 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -6,7 +6,7 @@ import core.file_interaction as fi from output_generators.logger import logger import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -14,8 +14,8 @@ def set_information_flows(dfd) -> dict: """Adds connections based on parsed config files. """ - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -40,7 +40,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index 7b68e6c..e897255 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -5,7 +5,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability from output_generators.logger import logger @@ -16,8 +16,8 @@ def set_information_flows(dfd) -> set: """Connects incoming endpoints, outgoing endpoints, and routings to information flows """ - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -39,7 +39,7 @@ def set_information_flows(dfd) -> set: information_flows = detect_stream_binders(microservices, information_flows, dfd) - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -184,8 +184,8 @@ def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints # incoming: (topic, microservice, (file, line, span)) # outgoing: (topic, microservice, asset, (file, line, span)) - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index 87aa583..d2cf577 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -7,7 +7,7 @@ import core.parse_files as parse import core.technology_switch as tech_sw import technology_specific_extractors.docker.dcr_entry as dcr -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability try: @@ -23,8 +23,8 @@ def set_microservices(dfd) -> dict: """Extracts the list of services from pom.xml files and sets the variable in the tmp-file. """ - if tmp.code2dfd_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.code2dfd_config["DFD"]["microservices"]) + if code2dfd_config.has_option("DFD", "microservices"): + microservices = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) else: microservices = dict() microservices_set = set() @@ -71,7 +71,7 @@ def set_microservices(dfd) -> dict: nested_microservices = check_nested_modules(module_dict) microservices_set.update(nested_microservices) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser return microservices @@ -81,7 +81,7 @@ def extract_dependencies(properties: set, pom_file) -> set: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -100,7 +100,7 @@ def extract_modules(pom_file: dict) -> list: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -146,7 +146,7 @@ def parse_properties_file(pom_path: str): # find properties file path = os.path.dirname(pom_path) - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -185,7 +185,7 @@ def extract_servicename_pom_file(pom_file) -> str: microservice = [False, False] file_name = pom_file["path"] - pom_path = os.path.join(tmp.code2dfd_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -224,7 +224,7 @@ def detect_microservice(file_path, dfd): path = file_path found_pom = False - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(path) diff --git a/technology_specific_extractors/nginx/ngn_entry.py b/technology_specific_extractors/nginx/ngn_entry.py index 9104698..8fa8678 100644 --- a/technology_specific_extractors/nginx/ngn_entry.py +++ b/technology_specific_extractors/nginx/ngn_entry.py @@ -4,7 +4,7 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -101,7 +101,7 @@ def detect_nginx(microservices: dict, information_flows: dict, external_componen id = 0 microservices[id] = dict() - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] docker_path = os.path.dirname(results[r]["path"]) if docker_path and local_repo_path: docker_path = os.path.relpath(docker_path, start=local_repo_path) diff --git a/technology_specific_extractors/prometheus/prm_entry.py b/technology_specific_extractors/prometheus/prm_entry.py index cb3ec24..4fe9fb9 100644 --- a/technology_specific_extractors/prometheus/prm_entry.py +++ b/technology_specific_extractors/prometheus/prm_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -59,7 +59,7 @@ def detect_connections(microservices: dict, information_flows: dict, dockerfile, """Parses config file to find connections to prometheus. """ - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] for line in dockerfile["content"]: if "ADD" in line: diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index f2e1340..f9a2bd8 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -5,7 +5,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability from output_generators.logger import logger @@ -17,8 +17,8 @@ def set_information_flows(dfd) -> set: if not used_in_application(): return - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -37,7 +37,7 @@ def set_information_flows(dfd) -> set: id = 0 information_flows[id] = new_information_flows[ni] - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -155,8 +155,8 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo # outgoing: (exchange, routingkey, microservice, (file, line, span)) # incoming: (queue, microservice, (file, line, span)) - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -304,7 +304,7 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo traceability.add_trace(trace) - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) return information_flows diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index c6303a1..e2713ae 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -4,7 +4,7 @@ import core.technology_switch as tech_sw from output_generators.logger import logger import output_generators.traceability as traceability -import core.config as tmp +from core.config import code2dfd_config def set_information_flows(dfd) -> dict: @@ -14,8 +14,8 @@ def set_information_flows(dfd) -> dict: if not used_in_application(): return - if tmp.code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.code2dfd_config["DFD"]["information_flows"]) + if code2dfd_config.has_option("DFD", "information_flows"): + information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) else: information_flows = dict() @@ -31,7 +31,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) + code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) return information_flows @@ -91,7 +91,7 @@ def get_incoming_endpoints(dfd) -> list: bracket_count = adjust_bracket_count(bracket_count, line) - tmp.code2dfd_config.set("DFD", "endpoints", str(endpoints)) + code2dfd_config.set("DFD", "endpoints", str(endpoints)) return endpoints @@ -135,7 +135,7 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): microservices[m]["tagged_values"].append(('Endpoints', list(ordered_endpoints[endpoint]))) else: microservices[m]["tagged_values"] = [('Endpoints', list(ordered_endpoints[endpoint]))] - tmp.code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) def get_outgoing_endpoints(information_flows: dict, dfd) -> set: diff --git a/technology_specific_extractors/spring_config/cnf_entry.py b/technology_specific_extractors/spring_config/cnf_entry.py index f2bc4b4..52d448a 100644 --- a/technology_specific_extractors/spring_config/cnf_entry.py +++ b/technology_specific_extractors/spring_config/cnf_entry.py @@ -4,7 +4,7 @@ import core.parse_files as parse import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -229,7 +229,7 @@ def parse_config_files(config_server: str, config_file_path: str, config_file_pa # external (other github repository) didn't work, look locally if config_file_path_local: - local_path = tmp.code2dfd_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] config_file_path_local = os.path.relpath(config_file_path_local, start=local_path) new_contents = fi.get_repo_contents_local(config_file_path_local) diff --git a/technology_specific_extractors/turbine/trb_entry.py b/technology_specific_extractors/turbine/trb_entry.py index fe233bd..d49d0a1 100644 --- a/technology_specific_extractors/turbine/trb_entry.py +++ b/technology_specific_extractors/turbine/trb_entry.py @@ -2,7 +2,7 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -155,7 +155,7 @@ def detect_turbine_stream(microservices: dict, information_flows: dict, dfd) -> found_pom = False - local_repo_path = tmp.code2dfd_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() From 2b0498586e88221cc0b39875e82c1c3e97496edc Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 22:44:54 +0200 Subject: [PATCH 08/10] refactor: DFD is a dictionary, do not use config parser to store data --- core/dfd_extraction.py | 171 +++++++++--------- core/technology_switch.py | 17 +- output_generators/codeable_model.py | 6 +- output_generators/json_architecture.py | 6 +- output_generators/json_edges.py | 3 +- output_generators/plaintext.py | 6 +- .../apache_httpd/aph_entry.py | 10 +- .../circuit_breaker/cbr_entry.py | 15 +- .../consul/cns_entry.py | 7 +- .../database_connections/dbc_entry.py | 41 ++--- .../databases/dbs_entry.py | 6 +- .../docker_compose/dcm_entry.py | 50 ++--- .../docker_compose/dcm_parser.py | 11 +- .../elasticsearch/ela_entry.py | 8 +- .../eureka/eur_entry.py | 12 +- .../feign_client/fgn_entry.py | 18 +- .../gradle/grd_entry.py | 11 +- .../grafana/grf_entry.py | 8 +- .../html/html_entry.py | 12 +- .../http_security/hts_entry.py | 11 +- .../hystrix/hsx_entry.py | 16 +- .../implicit_connections/imp_entry.py | 13 +- .../kafka/kfk_entry.py | 41 ++--- .../kibana/kib_entry.py | 9 +- .../load_balancer/lob_entry.py | 8 +- .../local_logging/llo_entry.py | 8 +- .../logstash/log_entry.py | 10 +- .../maven/mvn_entry.py | 10 +- .../nginx/ngn_entry.py | 10 +- .../plaintext_credentials/plc_entry.py | 7 +- .../prometheus/prm_entry.py | 8 +- .../rabbitmq/rmq_entry.py | 42 ++--- .../repository_rest_resource/rrr_entry.py | 6 +- .../resttemplate/rst_entry.py | 14 +- .../ribbon/rib_entry.py | 6 +- .../itf_entry.py | 8 +- .../spring_admin/sad_entry.py | 9 +- .../spring_config/cnf_entry.py | 10 +- .../spring_encryption/enc_entry.py | 8 +- .../spring_gateway/sgt_entry.py | 10 +- .../spring_oauth/soa_entry.py | 8 +- .../ssl/ssl_entry.py | 5 +- .../turbine/trb_entry.py | 7 +- .../zipkin/zip_entry.py | 13 +- .../zookeeper/zoo_entry.py | 11 +- .../zuul/zul_entry.py | 12 +- 46 files changed, 367 insertions(+), 371 deletions(-) diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index cf9143e..353f590 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -1,4 +1,3 @@ -import ast from datetime import datetime from itertools import combinations import os @@ -48,8 +47,6 @@ from technology_specific_extractors.zookeeper.zoo_entry import detect_zookeeper from technology_specific_extractors.zuul.zul_entry import detect_zuul -from core.DFD import CDFD - def perform_analysis(): """ @@ -57,7 +54,6 @@ def perform_analysis(): """ local_path = code2dfd_config.get("Repository", "local_path") url_path = code2dfd_config.get("Repository", "url") - code2dfd_config.add_section("DFD") os.makedirs(local_path, exist_ok=True) repository = Repository(path_to_repo=url_path, clone_repo_to=local_path) @@ -83,57 +79,46 @@ def perform_analysis(): def DFD_extraction(): """Main function for the extraction, calling all technology-specific extractors, managing output etc. """ - dfd = CDFD("TestDFD") - - microservices, information_flows, external_components = dict(), dict(), dict() + dfd = { + "microservices": dict(), + "information_flows": dict(), + "external_components": dict() + } - microservices = tech_sw.get_microservices(dfd) + tech_sw.get_microservices(dfd) - microservices = detect_databases(microservices) - microservices = overwrite_port(microservices) - microservices = detect_ssl_services(microservices) + detect_databases(dfd) + overwrite_port(dfd) + detect_ssl_services(dfd) print("Extracted services from build- and IaC-files") # Parse internal and external configuration files - microservices, information_flows, external_components = detect_spring_config(microservices, information_flows, external_components, dfd) - microservices = detect_eureka_server_only(microservices, dfd) - microservices = overwrite_port(microservices) + detect_spring_config(dfd) + detect_eureka_server_only(dfd) + overwrite_port(dfd) # Classify brokers (needed for information flows) - microservices = classify_brokers(microservices) + classify_brokers(dfd) # Check authentication information of services - microservices = detect_authentication_scopes(microservices, dfd) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + detect_authentication_scopes(dfd) # Get information flows - code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - - new_information_flows = tech_sw.get_information_flows(dfd) - external_components = ast.literal_eval(code2dfd_config["DFD"]["external_components"]) - - # Merge old and new - for new_flow in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = dict() - information_flows[id] = new_information_flows[new_flow] + tech_sw.get_information_flows(dfd) + print("Extracted information flows from API-calls, message brokers, and database connections") # Detect everything else / execute all technology implementations print("Classifying all services") - microservices = tech_sw.get_microservices(dfd) - microservices, information_flows, external_components = classify_microservices(microservices, information_flows, external_components, dfd) + classify_microservices(dfd) # Merging print("Merging duplicate items") - merge_duplicate_flows(information_flows) + information_flows = merge_duplicate_flows(dfd["information_flows"]) - merge_duplicate_nodes(microservices) - merge_duplicate_nodes(external_components) + microservices = merge_duplicate_nodes(dfd["microservices"]) + external_components = merge_duplicate_nodes(dfd["external_components"]) merge_duplicate_annotations(microservices) merge_duplicate_annotations(information_flows) @@ -141,80 +126,76 @@ def DFD_extraction(): print("Cleaning database connections") - clean_database_connections(microservices, information_flows) + dfd = { + "microservices": microservices, + "information_flows": information_flows, + "external_components": external_components + } + + clean_database_connections(dfd) # Printing print("\nFinished extraction") # Saving - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - - plaintext.write_plaintext(microservices, information_flows, external_components) - codeable_models, codeable_models_path = codeable_model.output_codeable_model(microservices, information_flows, external_components) + plaintext.write_plaintext(dfd) + codeable_models, codeable_models_path = codeable_model.output_codeable_model(dfd) traceability_content = traceability.output_traceability() visualizer.output_png(codeable_models_path) - json_edges.generate_json_edges(information_flows) - json_architecture.generate_json_architecture(microservices, information_flows, external_components) - - #calculate_metrics.calculate_single_system(repo_path) - - #check_traceability.check_traceability(microservices, information_flows, external_components, traceability_content) + json_edges.generate_json_edges(dfd) + json_architecture.generate_json_architecture(dfd) return codeable_models, traceability_content -def classify_brokers(microservices: dict) -> dict: +def classify_brokers(dfd: dict): """Classifies kafka and rabbitmq servers, because they are needed for the information flows. """ - microservices = detect_rabbitmq_server(microservices) - microservices = detect_kafka_server(microservices) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - return microservices + detect_rabbitmq_server(dfd) + detect_kafka_server(dfd) -def classify_microservices(microservices: dict, information_flows: dict, external_components: dict, dfd) -> tuple[dict, dict, dict]: +def classify_microservices(dfd): """Tries to determine the microservice's funcitonality. """ - microservices, information_flows = detect_eureka(microservices, information_flows, dfd) - microservices, information_flows, external_components = detect_zuul(microservices, information_flows, external_components, dfd) - microservices, information_flows, external_components = detect_spring_cloud_gateway(microservices, information_flows, external_components, dfd) - microservices, information_flows = detect_spring_oauth(microservices, information_flows, dfd) - microservices, information_flows = detect_consul(microservices, information_flows, dfd) - microservices, information_flows = detect_hystrix_dashboard(microservices, information_flows, dfd) - microservices, information_flows = detect_turbine(microservices, information_flows, dfd) - microservices, information_flows = detect_local_logging(microservices, information_flows, dfd) - microservices, information_flows = detect_zipkin_server(microservices, information_flows, dfd) - microservices, information_flows = detect_spring_admin_server(microservices, information_flows, dfd) - microservices, information_flows = detect_prometheus_server(microservices, information_flows, dfd) - microservices, information_flows = detect_circuit_breakers(microservices, information_flows, dfd) - microservices, information_flows = detect_load_balancers(microservices, information_flows, dfd) - microservices, information_flows = detect_ribbon_load_balancers(microservices, information_flows, dfd) - microservices, information_flows = detect_hystrix_circuit_breakers(microservices, information_flows, dfd) - microservices, information_flows = detect_zookeeper(microservices, information_flows, dfd) - microservices, information_flows = detect_kibana(microservices, information_flows, dfd) - microservices, information_flows = detect_elasticsearch(microservices, information_flows, dfd) - microservices, information_flows, external_components = detect_logstash(microservices, information_flows, external_components, dfd) - microservices, information_flows, external_components = detect_nginx(microservices, information_flows, external_components, dfd) - microservices, information_flows = detect_grafana(microservices, information_flows, dfd) - microservices, information_flows = detect_spring_encryption(microservices, information_flows, dfd) - microservices = detect_endpoints(microservices, dfd) - - microservices, information_flows, external_components = detect_miscellaneous(microservices, information_flows, external_components) - microservices, information_flows, external_components = detect_apachehttpd_webserver(microservices, information_flows, external_components, dfd) - microservices = classify_internal_infrastructural(microservices) - microservices = set_plaintext_credentials(microservices) - - return microservices, information_flows, external_components - - -def overwrite_port(microservices: dict) -> dict: + detect_eureka(dfd) + detect_zuul(dfd) + detect_spring_cloud_gateway(dfd) + detect_spring_oauth(dfd) + detect_consul(dfd) + detect_hystrix_dashboard(dfd) + detect_turbine(dfd) + detect_local_logging(dfd) + detect_zipkin_server(dfd) + detect_spring_admin_server(dfd) + detect_prometheus_server(dfd) + detect_circuit_breakers(dfd) + detect_load_balancers(dfd) + detect_ribbon_load_balancers(dfd) + detect_hystrix_circuit_breakers(dfd) + detect_zookeeper(dfd) + detect_kibana(dfd) + detect_elasticsearch(dfd) + detect_logstash(dfd) + detect_nginx(dfd) + detect_grafana(dfd) + detect_spring_encryption(dfd) + detect_endpoints(dfd) + + detect_miscellaneous(dfd) + detect_apachehttpd_webserver(dfd) + classify_internal_infrastructural(dfd) + set_plaintext_credentials(dfd) + + +def overwrite_port(dfd: dict): """Writes port from properties to tagged vallues. """ + microservices = dfd["microservices"] + for microservice in microservices.values(): for prop in microservice.get("properties", []): if prop[0] == "port": @@ -241,13 +222,17 @@ def overwrite_port(microservices: dict) -> dict: traceability.add_trace(trace) microservice["tagged_values"] = microservice.get("tagged_values", list()) + [("Port", port)] - return microservices + dfd["microservices"] = microservices -def detect_miscellaneous(microservices: dict, information_flows: dict, external_components: dict) -> tuple[dict, dict, dict]: +def detect_miscellaneous(dfd: dict): """Goes through properties extracted for each service to check for some things that don't fit anywhere else (mail servers, external websites, etc.). """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + for microservice in microservices.values(): for prop in microservice.get("properties", []): # external mail server @@ -394,7 +379,9 @@ def detect_miscellaneous(microservices: dict, information_flows: dict, external_ traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def merge_duplicate_flows(information_flows: dict): @@ -432,6 +419,8 @@ def merge_duplicate_flows(information_flows: dict): for k in to_delete: del information_flows[k] + return information_flows + def merge_duplicate_nodes(nodes: dict): """Merge duplicate nodes @@ -459,6 +448,8 @@ def merge_duplicate_nodes(nodes: dict): for k in to_delete: del nodes[k] + return nodes + def merge_duplicate_annotations(collection: dict): """Merge annotations of all items diff --git a/core/technology_switch.py b/core/technology_switch.py index af9c642..639770e 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -1,5 +1,3 @@ -import ast - from output_generators.logger import logger import technology_specific_extractors.database_connections.dbc_entry as dbc import technology_specific_extractors.docker_compose.dcm_entry as dcm @@ -11,7 +9,6 @@ import technology_specific_extractors.maven.mvn_entry as mvn import technology_specific_extractors.rabbitmq.rmq_entry as rmq import technology_specific_extractors.resttemplate.rst_entry as rst -from core.config import code2dfd_config COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, @@ -22,31 +19,29 @@ def get_microservices(dfd) -> dict: """Calls get_microservices from correct container technology or returns existing list. """ - if code2dfd_config.has_option("DFD", "microservices"): - return ast.literal_eval(code2dfd_config["DFD"]["microservices"]) + if len(dfd["microservices"]) > 0: + return dfd["microservices"] else: logger.info("Microservices not set yet, start extraction") mvn.set_microservices(dfd) grd.set_microservices(dfd) dcm.set_microservices(dfd) - if code2dfd_config.has_option("DFD", "microservices"): - return ast.literal_eval(code2dfd_config["DFD"]["microservices"]) + return dfd["microservices"] def get_information_flows(dfd) -> dict: """Calls get_information_flows from correct communication technology. """ - if code2dfd_config.has_option("DFD", "information_flows"): - return ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) + if len(dfd["information_flows"]) > 0: + return dfd["information_flows"] else: logger.info("Information flows not set yet, start extraction") for func in COMMUNICATIONS_TECH_LIST.values(): func.set_information_flows(dfd) - if code2dfd_config.has_option("DFD", "information_flows"): - return ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) + return dfd["information_flows"] def detect_microservice(file_path: str, dfd) -> str: diff --git a/output_generators/codeable_model.py b/output_generators/codeable_model.py index 9ad3139..46739c3 100644 --- a/output_generators/codeable_model.py +++ b/output_generators/codeable_model.py @@ -6,9 +6,13 @@ # the used metamodel is microservice_dfds_metamodel.py -def output_codeable_model(microservices, information_flows, external_components): +def output_codeable_model(dfd): """Entry function to creation of codeable models. Calls all necessary helper functions and outputs the codeable model""" + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + parts = Path(code2dfd_config["Analysis Settings"]["output_path"]).parts model_name = f"{parts[-2]}_{parts[-1]}" diff --git a/output_generators/json_architecture.py b/output_generators/json_architecture.py index 9b7ab15..093c8b4 100644 --- a/output_generators/json_architecture.py +++ b/output_generators/json_architecture.py @@ -5,10 +5,14 @@ from core.config import code2dfd_config -def generate_json_architecture(microservices: dict, information_flows: dict, external_components: dict): +def generate_json_architecture(dfd): """Creates JSON file that contains the complete extracted architecture. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + full_dict = {"microservices": list(microservices.values()), "information_flows": list(information_flows.values()), "external_components": list(external_components.values())} diff --git a/output_generators/json_edges.py b/output_generators/json_edges.py index b3b74e6..2a8dc6c 100644 --- a/output_generators/json_edges.py +++ b/output_generators/json_edges.py @@ -10,9 +10,10 @@ from core.config import code2dfd_config -def generate_json_edges(information_flows: dict): +def generate_json_edges(dfd: dict): """Creates JSON file that contains a list of plain information flows, without annotations. """ + information_flows = dfd["information_flows"] edges_list = list() for i in information_flows.keys(): diff --git a/output_generators/plaintext.py b/output_generators/plaintext.py index a999970..6ca8f1b 100644 --- a/output_generators/plaintext.py +++ b/output_generators/plaintext.py @@ -4,7 +4,11 @@ from core.config import code2dfd_config -def write_plaintext(microservices, information_flows, external_components): +def write_plaintext(dfd): + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_results.txt" diff --git a/technology_specific_extractors/apache_httpd/aph_entry.py b/technology_specific_extractors/apache_httpd/aph_entry.py index aa82a73..76ba34e 100644 --- a/technology_specific_extractors/apache_httpd/aph_entry.py +++ b/technology_specific_extractors/apache_httpd/aph_entry.py @@ -7,14 +7,20 @@ import output_generators.traceability as traceability -def detect_apachehttpd_webserver(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_apachehttpd_webserver(dfd): """Detects apachehttpd webservers and routes if possible. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + microservices, information_flows, external_components = detect_via_docker(microservices, information_flows, external_components, dfd) microservices, information_flows, external_components = detect_via_proxypass(microservices, information_flows, external_components, dfd) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def detect_via_docker(microservices: dict, information_flows: dict, external_components: dict, dfd): diff --git a/technology_specific_extractors/circuit_breaker/cbr_entry.py b/technology_specific_extractors/circuit_breaker/cbr_entry.py index 5007e59..3d31a07 100644 --- a/technology_specific_extractors/circuit_breaker/cbr_entry.py +++ b/technology_specific_extractors/circuit_breaker/cbr_entry.py @@ -2,10 +2,13 @@ import core.technology_switch as tech_sw -def detect_circuit_breakers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_circuit_breakers(dfd): """Find circuit breakers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableCircuitBreaker") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -50,11 +53,5 @@ def detect_circuit_breakers(microservices: dict, information_flows: dict, dfd) - else: information_flows[i]["tagged_values"] = [circuit_breaker_tuple] - return microservices, information_flows - - -def detect_circuit_breaker_tech(path): - - circuit_breaker_tuple = False - - return circuit_breaker_tuple + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/consul/cns_entry.py b/technology_specific_extractors/consul/cns_entry.py index fc78955..450d4b6 100644 --- a/technology_specific_extractors/consul/cns_entry.py +++ b/technology_specific_extractors/consul/cns_entry.py @@ -1,10 +1,12 @@ import output_generators.traceability as traceability -def detect_consul(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_consul(dfd): """Detects Consul server and clients (service discover, monitoring, configuration). """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] # Server consul_server = set() for m in microservices.keys(): @@ -43,4 +45,5 @@ def detect_consul(microservices: dict, information_flows: dict, dfd) -> dict: traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index 40a9b31..6dbe941 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -1,40 +1,21 @@ -import ast - import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw from core.config import code2dfd_config import output_generators.traceability as traceability -def set_information_flows(dfd) -> set: +def set_information_flows(dfd): """Goes through services and checks if there are connections to databases. """ - - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - if code2dfd_config.has_option("DFD", "external_components"): - external_components = ast.literal_eval(code2dfd_config["DFD"]["external_components"]) - else: - external_components = dict() - microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - return microservices, information_flows - - -def get_information_flows(microservices: dict, information_flows: dict, external_components: dict) -> dict: - - microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def check_properties(microservices: dict, information_flows: dict, external_components: dict) -> dict: @@ -220,8 +201,6 @@ def check_properties(microservices: dict, information_flows: dict, external_comp except: information_flows[id]["tagged_values"] = [("Username", username.strip())] - code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - trace = dict() trace["item"] = "database-" + str(microservices[m]["name"]) + " -> " + microservices[m]["name"] trace["file"] = trace_info[0] @@ -233,10 +212,13 @@ def check_properties(microservices: dict, information_flows: dict, external_comp return microservices, information_flows, external_components -def clean_database_connections(microservices: dict, information_flows: dict): +def clean_database_connections(dfd: dict): """Removes database connections in wrong direction, which can occur from docker compose. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + for microservice in microservices.values(): if microservice["type"] == "database_component": to_purge = set() @@ -245,3 +227,6 @@ def clean_database_connections(microservices: dict, information_flows: dict): to_purge.add(i) for p in to_purge: del information_flows[p] + + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/databases/dbs_entry.py b/technology_specific_extractors/databases/dbs_entry.py index 21f078b..095422e 100644 --- a/technology_specific_extractors/databases/dbs_entry.py +++ b/technology_specific_extractors/databases/dbs_entry.py @@ -1,10 +1,12 @@ import core.file_interaction as fi import output_generators.traceability as traceability -def detect_databases(microservices: dict) -> dict: + +def detect_databases(dfd: dict): """Detects databases. """ + microservices = dfd["microservices"] for m in microservices.keys(): database = False if "image" in microservices[m]: @@ -34,7 +36,7 @@ def detect_databases(microservices: dict) -> dict: else: microservices = detect_via_docker(microservices, m) - return microservices + dfd["microservices"] = microservices def detect_via_docker(microservices: dict, m: int) -> dict: diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index ad1fbbc..cc2dabd 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -1,4 +1,3 @@ -import ast import os import core.file_interaction as fi @@ -27,23 +26,23 @@ def set_microservices(dfd) -> None: if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose*") if len(raw_files) == 0: - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservices = clean_pom_names(microservices) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return docker_compose_content = raw_files[0]["content"] - microservices_set, properties_dict = dcm_parser.extract_microservices(docker_compose_content, raw_files[0]["path"]) + microservices_set, properties_dict = dcm_parser.extract_microservices(docker_compose_content, raw_files[0]["path"], dfd) if not microservices_set: - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservices = clean_pom_names(microservices) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return - microservices = dictionarify(microservices_set, properties_dict) + microservices = dictionarify(microservices_set, properties_dict, dfd["microservices"]) microservices = clean_pom_names(microservices) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices def clean_pom_names(microservices: dict) -> dict: @@ -56,15 +55,10 @@ def clean_pom_names(microservices: dict) -> dict: return microservices -def dictionarify(elements_set: set, properties_dict: dict) -> dict: +def dictionarify(elements_set: set, properties_dict: dict, microservices: dict) -> dict: """Turns set of services into dictionary. """ - if code2dfd_config.has_option("DFD", "microservices"): - elements = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) - else: - elements = dict() - for e in elements_set: try: properties = properties_dict[e[0]] @@ -87,17 +81,17 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: trace["span"] = e[3][3] traceability.add_trace(trace) try: - id = max(elements.keys()) + 1 + id = max(microservices.keys()) + 1 except: id = 0 - elements[id] = dict() + microservices[id] = dict() - elements[id]["name"] = e[0] - elements[id]["image"] = e[1] - elements[id]["type"] = e[2] - elements[id]["properties"] = properties - elements[id]["stereotype_instances"] = stereotypes - elements[id]["tagged_values"] = tagged_values + microservices[id]["name"] = e[0] + microservices[id]["image"] = e[1] + microservices[id]["type"] = e[2] + microservices[id]["properties"] = properties + microservices[id]["stereotype_instances"] = stereotypes + microservices[id]["tagged_values"] = tagged_values trace = dict() trace["item"] = e[0]#.replace("pom_", "") @@ -106,7 +100,7 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: trace["span"] = e[4][2] traceability.add_trace(trace) - return elements + return microservices def set_information_flows(dfd): @@ -115,12 +109,8 @@ def set_information_flows(dfd): global docker_compose_content - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] # Download docker-compose file if not docker_compose_content: @@ -134,9 +124,7 @@ def set_information_flows(dfd): docker_compose_content = raw_files[0]["content"] information_flows = dcm_parser.extract_information_flows(docker_compose_content, microservices, information_flows) - - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def get_environment_variables(docker_compose_file_URL: str) -> set: diff --git a/technology_specific_extractors/docker_compose/dcm_parser.py b/technology_specific_extractors/docker_compose/dcm_parser.py index 14c5b8a..5fae87a 100644 --- a/technology_specific_extractors/docker_compose/dcm_parser.py +++ b/technology_specific_extractors/docker_compose/dcm_parser.py @@ -1,10 +1,8 @@ -import ast import re from pathlib import Path import ruamel.yaml -import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env from core.config import code2dfd_config @@ -55,7 +53,7 @@ def construct_scalar(self, node): # end of external code -def extract_microservices(file_content, file_name) -> set: +def extract_microservices(file_content, file_name, dfd) -> set: """ Extracts the list of microservices from the docker-compose file autonomously, i.e. without asking for user-input in case of errors. """ @@ -67,10 +65,7 @@ def extract_microservices(file_content, file_name) -> set: image = False build = False - if code2dfd_config.has_option("DFD", "microservices"): - microservices_dict = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) - else: - microservices_dict= dict() + microservices_dict = dfd["microservices"] microservices_set = set() properties_dict = dict() @@ -621,7 +616,7 @@ def extract_microservices(file_content, file_name) -> set: else: microservices_dict[correct_id]["properties"] = properties - code2dfd_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) + dfd["microservices"] = microservices_dict return microservices_set, properties_dict diff --git a/technology_specific_extractors/elasticsearch/ela_entry.py b/technology_specific_extractors/elasticsearch/ela_entry.py index 26e0763..30fde43 100644 --- a/technology_specific_extractors/elasticsearch/ela_entry.py +++ b/technology_specific_extractors/elasticsearch/ela_entry.py @@ -1,10 +1,13 @@ import output_generators.traceability as traceability -def detect_elasticsearch(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_elasticsearch(dfd): """Detects elasticsearch services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + elasticsearch = False for m in microservices.keys(): if "elasticsearch:" in microservices[m]["image"]: @@ -52,4 +55,5 @@ def detect_elasticsearch(microservices: dict, information_flows: dict, dfd) -> d information_flows.pop(p) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/eureka/eur_entry.py b/technology_specific_extractors/eureka/eur_entry.py index 7bdb895..72f0e48 100644 --- a/technology_specific_extractors/eureka/eur_entry.py +++ b/technology_specific_extractors/eureka/eur_entry.py @@ -3,13 +3,15 @@ import output_generators.traceability as traceability -def detect_eureka(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_eureka(dfd): """Detects Eureka servers if there are any. """ # Server (/microservice classification) results = fi.search_keywords("@EnableEurekaServer") + microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] eureka_server = False for r in results.keys(): eureka_server = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -78,7 +80,8 @@ def detect_eureka(microservices: dict, information_flows: dict, dfd) -> dict: traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def is_eureka(microservice: tuple) -> bool: @@ -95,8 +98,9 @@ def is_eureka(microservice: tuple) -> bool: return False -def detect_eureka_server_only(microservices: dict, dfd): +def detect_eureka_server_only(dfd: dict): + microservices = dfd["microservices"] results = fi.search_keywords("@EnableEurekaServer") eureka_servers = set() for r in results.keys(): @@ -114,4 +118,4 @@ def detect_eureka_server_only(microservices: dict, dfd): except: microservices[m]["tagged_values"] = ("Service Discovery", "Eureka") - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index b4ea4df..309e467 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -1,21 +1,15 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw -from core.config import code2dfd_config import output_generators.traceability as traceability -def set_information_flows(dfd) -> dict: +def set_information_flows(dfd): """Detects uses of Feign Client in the code. """ microservices = tech_sw.get_microservices(dfd) - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() + information_flows = dfd["information_flows"] # check for circuit breaker results = fi.search_keywords("@EnableFeignClients") # content, name, path @@ -94,8 +88,7 @@ def set_information_flows(dfd) -> dict: traceability.add_trace(trace) - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def is_microservice(service: str, dfd) -> bool: @@ -111,8 +104,3 @@ def is_microservice(service: str, dfd) -> bool: is_microservice = True return is_microservice - - - - -# diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index 0a62032..dc5dd0f 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -1,6 +1,4 @@ -import ast import os -from pathlib import Path import core.file_interaction as fi from output_generators.logger import logger @@ -17,10 +15,7 @@ def set_microservices(dfd) -> dict: if not used_in_application(): return False - if code2dfd_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) - else: - microservices = dict() + microservices = dfd["microservices"] gradle_files = fi.get_file_as_lines("build.gradle") for gf in gradle_files.keys(): @@ -56,9 +51,7 @@ def set_microservices(dfd) -> dict: except: pass - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - - return microservices + dfd["microservices"] = microservices def used_in_application() -> bool: diff --git a/technology_specific_extractors/grafana/grf_entry.py b/technology_specific_extractors/grafana/grf_entry.py index bfda61c..c436fd0 100644 --- a/technology_specific_extractors/grafana/grf_entry.py +++ b/technology_specific_extractors/grafana/grf_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_grafana(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_grafana(dfd): """Detects grafana server and connections. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + grafana_server = str() results = fi.search_keywords("grafana/grafana") @@ -31,4 +34,5 @@ def detect_grafana(microservices: dict, information_flows: dict, dfd) -> dict: trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index 15f6fbe..4c3ee10 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -1,9 +1,6 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw import output_generators.traceability as traceability -from core.config import code2dfd_config def set_information_flows(dfd): @@ -11,11 +8,7 @@ def set_information_flows(dfd): """ microservices = tech_sw.get_microservices(dfd) - - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() + information_flows = dfd["information_flows"] results = fi.search_keywords("href") for r in results.keys(): @@ -52,5 +45,4 @@ def set_information_flows(dfd): except: pass - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/http_security/hts_entry.py b/technology_specific_extractors/http_security/hts_entry.py index ad1393e..c3722e0 100644 --- a/technology_specific_extractors/http_security/hts_entry.py +++ b/technology_specific_extractors/http_security/hts_entry.py @@ -3,14 +3,12 @@ import output_generators.traceability as traceability -def detect_authentication_scopes(microservices: dict, dfd) -> dict: +def detect_authentication_scopes(dfd: dict): """Detects authentication scopes via HttpSecurity configurations. """ configuration_tuples = detect_configurations(dfd) - microservices = interpret_configurations(microservices, configuration_tuples) - - return microservices + interpret_configurations(dfd, configuration_tuples) def detect_configurations(dfd): @@ -64,10 +62,11 @@ def detect_configurations(dfd): return configuration_tuples -def interpret_configurations(microservices: dict, configuration_tuples: list) -> dict: +def interpret_configurations(dfd: dict, configuration_tuples: list) -> dict: """Translates configurations into stereotypes and tagged values. """ + microservices = dfd["microservices"] for configuration_tuple in configuration_tuples: stereotypes, tagged_values = list(), list() # create stereotypes and tagged_values @@ -140,4 +139,4 @@ def interpret_configurations(microservices: dict, configuration_tuples: list) -> except: microservices[m]["tagged_values"] = tagged_values - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/hystrix/hsx_entry.py b/technology_specific_extractors/hystrix/hsx_entry.py index fbaccf0..1adc143 100644 --- a/technology_specific_extractors/hystrix/hsx_entry.py +++ b/technology_specific_extractors/hystrix/hsx_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_hystrix_dashboard(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_hystrix_dashboard(dfd): """Detects hystrix monitoring dashboards . """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableHystrixDashboard") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -31,13 +34,17 @@ def detect_hystrix_dashboard(microservices: dict, information_flows: dict, dfd) trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows -def detect_hystrix_circuit_breakers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_hystrix_circuit_breakers(dfd): """Detects HystrixCommand. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableHystrix") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -62,4 +69,5 @@ def detect_hystrix_circuit_breakers(microservices: dict, information_flows: dict trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index b51fb9e..c11ef20 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -1,4 +1,3 @@ -import ast import os import yaml @@ -6,20 +5,15 @@ import core.file_interaction as fi from output_generators.logger import logger import core.technology_switch as tech_sw -from core.config import code2dfd_config import output_generators.traceability as traceability -def set_information_flows(dfd) -> dict: +def set_information_flows(dfd): """Adds connections based on parsed config files. """ - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] # Weavescope new_information_flows = weavescope(microservices) @@ -40,8 +34,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def weavescope(microservices): diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index e897255..ba3a423 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -5,43 +5,36 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -from core.config import code2dfd_config import output_generators.traceability as traceability from output_generators.logger import logger kafka_server = str() -def set_information_flows(dfd) -> set: +def set_information_flows(dfd): """Connects incoming endpoints, outgoing endpoints, and routings to information flows """ - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] incoming_endpoints = get_incoming_endpoints(dfd) outgoing_endpoints = get_outgoing_endpoints(dfd) - new_information_flows = match_incoming_to_outgoing_endpoints(microservices, incoming_endpoints, outgoing_endpoints) + new_information_flows = match_incoming_to_outgoing_endpoints(microservices, information_flows, incoming_endpoints, outgoing_endpoints) # merge old and new flows - for ni in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = new_information_flows[ni] + if new_information_flows is not information_flows: + for ni in new_information_flows.keys(): + try: + id = max(information_flows.keys()) + 1 + except: + id = 0 + information_flows[id] = new_information_flows[ni] information_flows = detect_stream_binders(microservices, information_flows, dfd) - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - - return information_flows + dfd["information_flows"] = information_flows def get_incoming_endpoints(dfd) -> set: @@ -178,17 +171,12 @@ def asset_is_input(variable: str, file, line_nr: int) -> bool: return False -def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints: set, outgoing_endpoints: set) -> dict: +def match_incoming_to_outgoing_endpoints(microservices: dict, information_flows: dict, incoming_endpoints: set, outgoing_endpoints: set) -> dict: """Finds information flows by regexing routing keys of outgoing endpoints to queues of incoming endpoints. """ # incoming: (topic, microservice, (file, line, span)) # outgoing: (topic, microservice, asset, (file, line, span)) - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - kafka_server = False for id in microservices.keys(): if ("Message Broker", "Kafka") in microservices[id]["tagged_values"]: @@ -310,12 +298,13 @@ def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints return information_flows -def detect_kafka_server(microservices: dict) -> dict: +def detect_kafka_server(dfd: dict) -> dict: """Detects and marks kafka server. """ global kafka_server + microservices = dfd["microservices"] raw_files = fi.get_file_as_yaml("docker-compose.yml") if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose.yaml") @@ -379,7 +368,7 @@ def detect_kafka_server(microservices: dict) -> dict: traceability.add_trace(trace) except: pass - return microservices + dfd["microservices"] = microservices def detect_stream_binders(microservices: dict, information_flows: dict, dfd) -> dict: diff --git a/technology_specific_extractors/kibana/kib_entry.py b/technology_specific_extractors/kibana/kib_entry.py index 02e27d6..b873d07 100644 --- a/technology_specific_extractors/kibana/kib_entry.py +++ b/technology_specific_extractors/kibana/kib_entry.py @@ -1,9 +1,13 @@ import output_generators.traceability as traceability -def detect_kibana(microservices: dict, information_flows: dict, dfd) -> dict: + +def detect_kibana(dfd): """Detects logstash services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + for m in microservices.keys(): if "kibana:" in microservices[m]["image"]: try: @@ -22,4 +26,5 @@ def detect_kibana(microservices: dict, information_flows: dict, dfd) -> dict: trace["span"] = "heuristic, based on image" traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/load_balancer/lob_entry.py b/technology_specific_extractors/load_balancer/lob_entry.py index 1670888..6029cb6 100644 --- a/technology_specific_extractors/load_balancer/lob_entry.py +++ b/technology_specific_extractors/load_balancer/lob_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_load_balancers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_load_balancers(dfd): """Find load balancers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@LoadBalanced") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -50,4 +53,5 @@ def detect_load_balancers(microservices: dict, information_flows: dict, dfd) -> else: information_flows[i]["tagged_values"] = [('Load Balancer', "Spring Cloud")] - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/local_logging/llo_entry.py b/technology_specific_extractors/local_logging/llo_entry.py index 5e508e7..09e3151 100644 --- a/technology_specific_extractors/local_logging/llo_entry.py +++ b/technology_specific_extractors/local_logging/llo_entry.py @@ -3,14 +3,18 @@ import output_generators.traceability as traceability -def detect_local_logging(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_local_logging(dfd): """Detects if a service performs local logging. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices = detect_loggerfactory(microservices, dfd) microservices = detect_lombok(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_loggerfactory(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/logstash/log_entry.py b/technology_specific_extractors/logstash/log_entry.py index 67d5a74..b02c808 100644 --- a/technology_specific_extractors/logstash/log_entry.py +++ b/technology_specific_extractors/logstash/log_entry.py @@ -1,10 +1,14 @@ import output_generators.traceability as traceability -def detect_logstash(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_logstash(dfd): """Detects logstash services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + # Service logstash = False trace_info = False @@ -155,4 +159,6 @@ def detect_logstash(microservices: dict, information_flows: dict, external_compo traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index d2cf577..0d149fa 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -1,4 +1,3 @@ -import ast import os import re @@ -23,10 +22,7 @@ def set_microservices(dfd) -> dict: """Extracts the list of services from pom.xml files and sets the variable in the tmp-file. """ - if code2dfd_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(code2dfd_config["DFD"]["microservices"]) - else: - microservices = dict() + microservices = dfd["microservices"] microservices_set = set() pom_files = fi.get_file_as_lines("pom.xml") @@ -71,9 +67,7 @@ def set_microservices(dfd) -> dict: nested_microservices = check_nested_modules(module_dict) microservices_set.update(nested_microservices) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser - - return microservices + dfd["microservices"] = microservices def extract_dependencies(properties: set, pom_file) -> set: diff --git a/technology_specific_extractors/nginx/ngn_entry.py b/technology_specific_extractors/nginx/ngn_entry.py index 8fa8678..f40a299 100644 --- a/technology_specific_extractors/nginx/ngn_entry.py +++ b/technology_specific_extractors/nginx/ngn_entry.py @@ -54,10 +54,14 @@ def construct_scalar(self, node): # end of external code -def detect_nginx(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_nginx(dfd): """Detects nginx web applications. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + web_app = False port = False correct_id = False @@ -279,4 +283,6 @@ def detect_nginx(microservices: dict, information_flows: dict, external_componen # Add flows between web_app and user information_flows = ext.add_user_connections(information_flows, web_app) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/plaintext_credentials/plc_entry.py b/technology_specific_extractors/plaintext_credentials/plc_entry.py index 923ad92..53e3829 100644 --- a/technology_specific_extractors/plaintext_credentials/plc_entry.py +++ b/technology_specific_extractors/plaintext_credentials/plc_entry.py @@ -1,12 +1,13 @@ - import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env -def set_plaintext_credentials(microservices: dict) -> dict: +def set_plaintext_credentials(dfd: dict): """Goes through properties and sets stereotype and tagged values, if plaintext credentials are found. """ + microservices = dfd["microservices"] + for m in microservices.keys(): plaintext_credentials = False tagged_values = set() @@ -63,4 +64,4 @@ def set_plaintext_credentials(microservices: dict) -> dict: else: microservices[m]["tagged_values"] = tagged_values - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/prometheus/prm_entry.py b/technology_specific_extractors/prometheus/prm_entry.py index 4fe9fb9..db2d39c 100644 --- a/technology_specific_extractors/prometheus/prm_entry.py +++ b/technology_specific_extractors/prometheus/prm_entry.py @@ -6,13 +6,17 @@ import output_generators.traceability as traceability -def detect_prometheus_server(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_prometheus_server(dfd): """Detects prometheus server and adds information flows. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices, information_flows = detect_server_docker(microservices, information_flows, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_server_docker(microservices: dict, information_flows: dict, dfd) -> dict: diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index f9a2bd8..3a1e5e9 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -1,11 +1,9 @@ -import ast import re import yaml import core.file_interaction as fi import core.technology_switch as tech_sw -from core.config import code2dfd_config import output_generators.traceability as traceability from output_generators.logger import logger @@ -17,28 +15,24 @@ def set_information_flows(dfd) -> set: if not used_in_application(): return - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - + information_flows = dfd["information_flows"] new_information_flows = dict() routings = get_routings() incoming_endpoints = get_incoming_endpoints(dfd) outgoing_endpoints = get_outgoing_endpoints(routings, dfd) - new_information_flows = match_incoming_to_outgoing_endpoints(incoming_endpoints, outgoing_endpoints, dfd) + new_information_flows = match_incoming_to_outgoing_endpoints(information_flows, incoming_endpoints, outgoing_endpoints, dfd) # merge old and new flows - for ni in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = new_information_flows[ni] - - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + if new_information_flows is not information_flows: + for flow in new_information_flows.values(): + try: + id = max(information_flows.keys()) + 1 + except: + id = 0 + information_flows[id] = flow + + dfd["information_flows"] = information_flows def used_in_application(): @@ -148,18 +142,13 @@ def get_outgoing_endpoints(routings: set, dfd) -> set: return outgoing_endpoints -def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpoints: set, dfd) -> dict: +def match_incoming_to_outgoing_endpoints(information_flows: dict, incoming_endpoints: set, outgoing_endpoints: set, dfd): """Finds information flows by regexing routing keys of outgoing endpoints to queues of incoming endpoints. """ # outgoing: (exchange, routingkey, microservice, (file, line, span)) # incoming: (queue, microservice, (file, line, span)) - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() - microservices = tech_sw.get_microservices(dfd) rabbit_server = False for id in microservices.keys(): @@ -304,14 +293,15 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo traceability.add_trace(trace) - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return information_flows -def detect_rabbitmq_server(microservices: dict) -> dict: +def detect_rabbitmq_server(dfd: dict) -> dict: """Detects RabbitMQ server. """ + microservices = dfd["microservices"] raw_files = fi.get_file_as_yaml("docker-compose.yml") if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose.yaml") @@ -395,4 +385,4 @@ def detect_rabbitmq_server(microservices: dict) -> dict: except: pass - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/repository_rest_resource/rrr_entry.py b/technology_specific_extractors/repository_rest_resource/rrr_entry.py index b283dcb..af1e57d 100644 --- a/technology_specific_extractors/repository_rest_resource/rrr_entry.py +++ b/technology_specific_extractors/repository_rest_resource/rrr_entry.py @@ -2,10 +2,12 @@ import core.technology_switch as tech_sw -def detect_endpoints(microservices: dict, dfd) -> dict: +def detect_endpoints(dfd): """Detects endpoints offered via @RepositoryRestResource """ + microservices = dfd["microservices"] + results = fi.search_keywords("@RepositoryRestResource") for r in results.keys(): endpoints = set() @@ -22,4 +24,4 @@ def detect_endpoints(microservices: dict, dfd) -> dict: microservices[m]["tagged_values"].append(("Endpoints", list(endpoints))) except: microservices[m]["tagged_values"] = [("Endpoints", list(endpoints))] - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index e2713ae..7b6a744 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -1,10 +1,7 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw from output_generators.logger import logger import output_generators.traceability as traceability -from core.config import code2dfd_config def set_information_flows(dfd) -> dict: @@ -14,10 +11,7 @@ def set_information_flows(dfd) -> dict: if not used_in_application(): return - if code2dfd_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(code2dfd_config["DFD"]["information_flows"]) - else: - information_flows = dict() + information_flows = dfd["information_flows"] incoming_endpoints = get_incoming_endpoints(dfd) add_endpoints_tagged_values(incoming_endpoints, dfd) @@ -31,8 +25,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - code2dfd_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def used_in_application(): @@ -91,7 +84,6 @@ def get_incoming_endpoints(dfd) -> list: bracket_count = adjust_bracket_count(bracket_count, line) - code2dfd_config.set("DFD", "endpoints", str(endpoints)) return endpoints @@ -135,7 +127,7 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): microservices[m]["tagged_values"].append(('Endpoints', list(ordered_endpoints[endpoint]))) else: microservices[m]["tagged_values"] = [('Endpoints', list(ordered_endpoints[endpoint]))] - code2dfd_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices def get_outgoing_endpoints(information_flows: dict, dfd) -> set: diff --git a/technology_specific_extractors/ribbon/rib_entry.py b/technology_specific_extractors/ribbon/rib_entry.py index ac80c86..c5a9ee2 100644 --- a/technology_specific_extractors/ribbon/rib_entry.py +++ b/technology_specific_extractors/ribbon/rib_entry.py @@ -2,13 +2,15 @@ import core.technology_switch as tech_sw -def detect_ribbon_load_balancers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_ribbon_load_balancers(dfd): """Detects load balancing via Ribbon. """ + microservices = dfd["microservices"] + microservices = detect_client_side(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices def detect_client_side(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/service_functionality_classification/itf_entry.py b/technology_specific_extractors/service_functionality_classification/itf_entry.py index 6a89228..55c06eb 100644 --- a/technology_specific_extractors/service_functionality_classification/itf_entry.py +++ b/technology_specific_extractors/service_functionality_classification/itf_entry.py @@ -1,6 +1,7 @@ import output_generators.traceability as traceability -def classify_internal_infrastructural(microservices: dict) -> dict: + +def classify_internal_infrastructural(dfd: dict): """Classifies processes as either internal or infrastructural. The latter if they are marked as one of the known infrastructural technologies. """ @@ -26,6 +27,8 @@ def classify_internal_infrastructural(microservices: dict) -> dict: "proxy" ] + microservices = dfd["microservices"] + for m in microservices.keys(): infrastructural = False if not "database" in microservices[m]["stereotype_instances"]: @@ -60,5 +63,4 @@ def classify_internal_infrastructural(microservices: dict) -> dict: trace["span"] = "heuristic" traceability.add_trace(trace) - - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/spring_admin/sad_entry.py b/technology_specific_extractors/spring_admin/sad_entry.py index 0e2c276..8068323 100644 --- a/technology_specific_extractors/spring_admin/sad_entry.py +++ b/technology_specific_extractors/spring_admin/sad_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_spring_admin_server(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_admin_server(dfd): """Detects Spring Admin Servers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableAdminServer") admin_server = False for r in results.keys(): @@ -141,5 +144,5 @@ def detect_spring_admin_server(microservices: dict, information_flows: dict, dfd traceability.add_trace(trace) - - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/spring_config/cnf_entry.py b/technology_specific_extractors/spring_config/cnf_entry.py index 52d448a..874ae31 100644 --- a/technology_specific_extractors/spring_config/cnf_entry.py +++ b/technology_specific_extractors/spring_config/cnf_entry.py @@ -8,17 +8,21 @@ import output_generators.traceability as traceability -def detect_spring_config(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_spring_config(dfd: dict): """Detects Spring Cloud Config server and connections to it. And parses config files. """ - + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] config_server, config_path = False, False microservices, config_server, config_path, config_file_path, config_repo_uri, config_server_ports, config_file_path_local = detect_config_server(microservices, dfd) if config_file_path or config_repo_uri or config_file_path_local: microservices, information_flows, external_components = parse_config_files(config_server, config_file_path, config_file_path_local, config_repo_uri, microservices, information_flows, external_components) microservices, information_flows = detect_config_clients(microservices, information_flows, config_server, config_server_ports) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def detect_config_server(microservices: dict, dfd): diff --git a/technology_specific_extractors/spring_encryption/enc_entry.py b/technology_specific_extractors/spring_encryption/enc_entry.py index 1ba796a..1e81a6a 100644 --- a/technology_specific_extractors/spring_encryption/enc_entry.py +++ b/technology_specific_extractors/spring_encryption/enc_entry.py @@ -2,15 +2,19 @@ import core.technology_switch as tech_sw -def detect_spring_encryption(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_encryption(dfd): """Detects use of Spring's crypto module encryption functions. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices = detect_passwordEncoder(microservices, dfd) microservices = detect_bytesEncryptor(microservices, dfd) microservices = detect_keyGenerator(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_passwordEncoder(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/spring_gateway/sgt_entry.py b/technology_specific_extractors/spring_gateway/sgt_entry.py index a31d27c..0a8aee5 100644 --- a/technology_specific_extractors/spring_gateway/sgt_entry.py +++ b/technology_specific_extractors/spring_gateway/sgt_entry.py @@ -4,10 +4,14 @@ import output_generators.traceability as traceability -def detect_spring_cloud_gateway(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_spring_cloud_gateway(dfd): """Detetcs Spring Cloud Gateway. """ + microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + server = False results = fi.search_keywords("spring-cloud-starter-gateway") for r in results.keys(): @@ -84,4 +88,6 @@ def detect_spring_cloud_gateway(microservices: dict, information_flows: dict, ex trace["span"] = prop[2][2] traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/spring_oauth/soa_entry.py b/technology_specific_extractors/spring_oauth/soa_entry.py index bc5484e..1d6480a 100644 --- a/technology_specific_extractors/spring_oauth/soa_entry.py +++ b/technology_specific_extractors/spring_oauth/soa_entry.py @@ -4,16 +4,20 @@ import output_generators.traceability as traceability -def detect_spring_oauth(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_oauth(dfd): """Detect Spring OAuth Server and connections to it. """ + microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] + microservices = detect_authorization_server(microservices, dfd) microservices = detect_resource_servers(microservices, dfd) microservices, information_flows = detect_token_server(microservices, information_flows, dfd) microservices = detect_preauthorized_methods(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_authorization_server(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/ssl/ssl_entry.py b/technology_specific_extractors/ssl/ssl_entry.py index 5009d5f..d51b668 100644 --- a/technology_specific_extractors/ssl/ssl_entry.py +++ b/technology_specific_extractors/ssl/ssl_entry.py @@ -1,8 +1,9 @@ -def detect_ssl_services(microservices: dict) -> dict: +def detect_ssl_services(dfd: dict): """Checks if services have ssl enabled. """ + microservices = dfd["microservices"] for m in microservices.keys(): for prop in microservices[m]["properties"]: if prop[0] == "ssl_enabled": @@ -23,4 +24,4 @@ def detect_ssl_services(microservices: dict) -> dict: except: microservices[m]["tagged_values"] = [("SSL Protocol", prop[1])] - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/turbine/trb_entry.py b/technology_specific_extractors/turbine/trb_entry.py index d49d0a1..cf5689e 100644 --- a/technology_specific_extractors/turbine/trb_entry.py +++ b/technology_specific_extractors/turbine/trb_entry.py @@ -6,15 +6,18 @@ import output_generators.traceability as traceability -def detect_turbine(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_turbine(dfd): """Detects turbine server. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] microservices = detect_turbine_server(microservices, dfd) microservices, information_flows = detect_turbineamqp(microservices, information_flows, dfd) microservices, information_flows = detect_turbine_stream(microservices, information_flows, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_turbine_server(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/zipkin/zip_entry.py b/technology_specific_extractors/zipkin/zip_entry.py index 900b723..d93eeb7 100644 --- a/technology_specific_extractors/zipkin/zip_entry.py +++ b/technology_specific_extractors/zipkin/zip_entry.py @@ -1,10 +1,13 @@ import output_generators.traceability as traceability -def detect_zipkin_server(microservices: dict, information_flows: dict, iterative=False) -> dict: +def detect_zipkin_server(dfd: dict, iterative=False): """Detects zipkin server and connections to it. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + zipkin_server_exists, connections_exist = False, False for m in microservices.keys(): @@ -98,8 +101,10 @@ def detect_zipkin_server(microservices: dict, information_flows: dict, iterative microservices[id_]["stereotype_instances"] = ["tracing_server"] microservices[id_]["tagged_values"] = [("Tracing Server", "Zipkin")] - + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows if not iterative: - microservices, information_flows = detect_zipkin_server(microservices, information_flows, True) + detect_zipkin_server(dfd, True) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/zookeeper/zoo_entry.py b/technology_specific_extractors/zookeeper/zoo_entry.py index d080305..74389ec 100644 --- a/technology_specific_extractors/zookeeper/zoo_entry.py +++ b/technology_specific_extractors/zookeeper/zoo_entry.py @@ -1,10 +1,10 @@ -import output_generators.traceability as traceability - - -def detect_zookeeper(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_zookeeper(dfd): """Detects ZooKeeper config services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + zookeeper_service = False # Service @@ -45,4 +45,5 @@ def detect_zookeeper(microservices: dict, information_flows: dict, dfd) -> dict: for p in to_purge: information_flows.pop(p) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/zuul/zul_entry.py b/technology_specific_extractors/zuul/zul_entry.py index ef5ff3a..9d7ac6b 100644 --- a/technology_specific_extractors/zuul/zul_entry.py +++ b/technology_specific_extractors/zuul/zul_entry.py @@ -1,11 +1,10 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import core.config as tmp import output_generators.traceability as traceability -def detect_zuul(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_zuul(dfd): """Detects Zuul gateway if there is one. """ @@ -13,6 +12,10 @@ def detect_zuul(microservices: dict, information_flows: dict, external_component results = fi.search_keywords("@EnableZuulServer") new_results = fi.search_keywords("@EnableZuulProxy") + microservices = tech_sw.get_microservices(dfd) + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + for r in new_results.keys(): try: id = max(results.keys()) + 1 @@ -112,5 +115,6 @@ def detect_zuul(microservices: dict, information_flows: dict, external_component except: information_flows[id]["tagged_values"] = [("Load Balancer", load_balancer)] - tmp.code2dfd_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components From fa26d9092ccaf5262bc05ce15c979ec01c2b0051 Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 22:59:12 +0200 Subject: [PATCH 09/10] refactor: do not use technology_switch to get services/flows --- core/dfd_extraction.py | 6 ++-- core/file_interaction.py | 35 +------------------ core/technology_switch.py | 27 +++++--------- .../database_connections/dbc_entry.py | 4 +-- .../docker_compose/dcm_entry.py | 5 ++- .../eureka/eur_entry.py | 2 +- .../feign_client/fgn_entry.py | 5 ++- .../gradle/grd_entry.py | 3 +- .../html/html_entry.py | 2 +- .../implicit_connections/imp_entry.py | 3 +- .../kafka/kfk_entry.py | 2 +- .../maven/mvn_entry.py | 3 +- .../rabbitmq/rmq_entry.py | 2 +- .../resttemplate/rst_entry.py | 8 ++--- .../spring_gateway/sgt_entry.py | 2 +- .../spring_oauth/soa_entry.py | 2 +- .../zuul/zul_entry.py | 2 +- 17 files changed, 32 insertions(+), 81 deletions(-) diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index 353f590..557770d 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -85,7 +85,8 @@ def DFD_extraction(): "external_components": dict() } - tech_sw.get_microservices(dfd) + print("Extracting microservices") + tech_sw.set_microservices(dfd) detect_databases(dfd) overwrite_port(dfd) @@ -104,7 +105,8 @@ def DFD_extraction(): detect_authentication_scopes(dfd) # Get information flows - tech_sw.get_information_flows(dfd) + print("Extracting information flows") + tech_sw.set_information_flows(dfd) print("Extracted information flows from API-calls, message brokers, and database connections") diff --git a/core/file_interaction.py b/core/file_interaction.py index 6923fff..527c8e8 100644 --- a/core/file_interaction.py +++ b/core/file_interaction.py @@ -4,7 +4,6 @@ from pathlib import Path, PurePosixPath from output_generators.logger import logger -import core.technology_switch as tech_sw from core.config import code2dfd_config @@ -179,38 +178,6 @@ def file_as_lines(path): return file_as_lines -def detect_microservice(file_path, dfd): - """Finds microservice that a file belongs to. - """ - - microservices_set = tech_sw.get_microservices(dfd) - microservices = [microservices_set[x]["name"] for x in microservices_set.keys()] - - file_path_parts = Path(file_path).parts - count = 0 - part = 0 - found = False - - while part < len(file_path_parts) and not found: - for m in microservices: - if m == file_path_parts[part]: - microservice = m - count += 1 - if count > 0: - found = True - part += 1 - if count == 1: - return microservice - else: - print("\tFound " + str(count) + " microservices for file " + str(file_path) +". \ - \n\tPlease choose microservice that the file belongs to: ") - i = 1 - for m in microservices: - print("\t[" + str(i) + "] " + str(m)) - i += 1 - return microservices[int(input("\n\t > ")) - 1] - - def find_variable(parameter: str, file) -> str: """ Looks for passed ´parameter´ in passed ´file´ or other files. """ @@ -287,7 +254,7 @@ def resolve_url(url: str, microservice: str, dfd) -> str: """Tries to resolve a url into one of the microserices. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] target_service = False if "http" in url: diff --git a/core/technology_switch.py b/core/technology_switch.py index 639770e..e246d85 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -1,4 +1,3 @@ -from output_generators.logger import logger import technology_specific_extractors.database_connections.dbc_entry as dbc import technology_specific_extractors.docker_compose.dcm_entry as dcm import technology_specific_extractors.feign_client.fgn_entry as fgn @@ -15,33 +14,23 @@ "Docker-Compose": dcm} -def get_microservices(dfd) -> dict: +def set_microservices(dfd): """Calls get_microservices from correct container technology or returns existing list. """ - if len(dfd["microservices"]) > 0: - return dfd["microservices"] - else: - logger.info("Microservices not set yet, start extraction") + mvn.set_microservices(dfd) + grd.set_microservices(dfd) + dcm.set_microservices(dfd) - mvn.set_microservices(dfd) - grd.set_microservices(dfd) - dcm.set_microservices(dfd) - return dfd["microservices"] - -def get_information_flows(dfd) -> dict: +def set_information_flows(dfd) -> dict: """Calls get_information_flows from correct communication technology. """ - if len(dfd["information_flows"]) > 0: - return dfd["information_flows"] - else: - logger.info("Information flows not set yet, start extraction") - for func in COMMUNICATIONS_TECH_LIST.values(): - func.set_information_flows(dfd) + for func in COMMUNICATIONS_TECH_LIST.values(): + func.set_information_flows(dfd) - return dfd["information_flows"] + return dfd["information_flows"] def detect_microservice(file_path: str, dfd) -> str: diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index 6dbe941..51c183c 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -1,13 +1,11 @@ import technology_specific_extractors.environment_variables as env -import core.technology_switch as tech_sw -from core.config import code2dfd_config import output_generators.traceability as traceability def set_information_flows(dfd): """Goes through services and checks if there are connections to databases. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] external_components = dfd["external_components"] diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index cc2dabd..bd4e343 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -2,7 +2,6 @@ import core.file_interaction as fi from output_generators.logger import logger -import core.technology_switch as tech_sw import technology_specific_extractors.docker_compose.dcm_parser as dcm_parser from core.config import code2dfd_config import output_generators.traceability as traceability @@ -109,7 +108,7 @@ def set_information_flows(dfd): global docker_compose_content - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] # Download docker-compose file @@ -160,7 +159,7 @@ def detect_microservice(file_path: str, dfd) -> str: """Detects, which service a file belongs to based on image given in docker-compose file and dockerfile belonging to file given as input. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservice = False dockerfile_path = False diff --git a/technology_specific_extractors/eureka/eur_entry.py b/technology_specific_extractors/eureka/eur_entry.py index 72f0e48..6bcaf76 100644 --- a/technology_specific_extractors/eureka/eur_entry.py +++ b/technology_specific_extractors/eureka/eur_entry.py @@ -10,7 +10,7 @@ def detect_eureka(dfd): # Server (/microservice classification) results = fi.search_keywords("@EnableEurekaServer") - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] eureka_server = False for r in results.keys(): diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index 309e467..4ff262d 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -7,8 +7,7 @@ def set_information_flows(dfd): """Detects uses of Feign Client in the code. """ - microservices = tech_sw.get_microservices(dfd) - + microservices = dfd["microservices"] information_flows = dfd["information_flows"] # check for circuit breaker @@ -98,7 +97,7 @@ def is_microservice(service: str, dfd) -> bool: if not service: return False is_microservice = False - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] for m in microservices.keys(): if service.casefold() == microservices[m]["name"].casefold(): is_microservice = True diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index dc5dd0f..6d7580e 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -3,7 +3,6 @@ import core.file_interaction as fi from output_generators.logger import logger import core.parse_files as parse -import core.technology_switch as tech_sw from core.config import code2dfd_config import output_generators.traceability as traceability @@ -123,7 +122,7 @@ def detect_microservice(file_path, dfd): return False microservice = [False, False] - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] found_gradle = False diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index 4c3ee10..42ad3d3 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -7,7 +7,7 @@ def set_information_flows(dfd): """Looks for connections between services via html sites / href's. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] results = fi.search_keywords("href") diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index c11ef20..5ed2e93 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -4,7 +4,6 @@ import core.file_interaction as fi from output_generators.logger import logger -import core.technology_switch as tech_sw import output_generators.traceability as traceability @@ -12,7 +11,7 @@ def set_information_flows(dfd): """Adds connections based on parsed config files. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] # Weavescope diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index ba3a423..2d40094 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -15,7 +15,7 @@ def set_information_flows(dfd): """Connects incoming endpoints, outgoing endpoints, and routings to information flows """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] incoming_endpoints = get_incoming_endpoints(dfd) diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index 0d149fa..beaed7a 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -4,7 +4,6 @@ import core.file_interaction as fi from output_generators.logger import logger import core.parse_files as parse -import core.technology_switch as tech_sw import technology_specific_extractors.docker.dcr_entry as dcr from core.config import code2dfd_config import output_generators.traceability as traceability @@ -213,7 +212,7 @@ def detect_microservice(file_path, dfd): """ microservice = [False, False] - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] path = file_path found_pom = False diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index 3a1e5e9..be71b9f 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -149,7 +149,7 @@ def match_incoming_to_outgoing_endpoints(information_flows: dict, incoming_endpo # outgoing: (exchange, routingkey, microservice, (file, line, span)) # incoming: (queue, microservice, (file, line, span)) - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] rabbit_server = False for id in microservices.keys(): if ("Message Broker", "RabbitMQ") in microservices[id]["tagged_values"]: diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index 7b6a744..7285645 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -111,7 +111,7 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): """Adds tagged values containing the endpoitns to the microservices. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] ordered_endpoints = dict() for endpoint_tuple in endpoint_tuples: @@ -134,7 +134,7 @@ def get_outgoing_endpoints(information_flows: dict, dfd) -> set: """Finds API calls from one service to another if restTemplate.exchange() is used in the application. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] if microservices != None: microservices = [microservices[x]["name"] for x in microservices.keys()] @@ -178,7 +178,7 @@ def get_outgoing_endpoints(information_flows: dict, dfd) -> set: def find_rst_variable(parameter: str, file: dict, line_nr: int, information_flows: dict, microservice: str, dfd, count=0): # check if service name in parameter, if yes, add flow directly - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] for m in microservices.keys(): if microservices[m]["name"] in parameter: try: @@ -303,7 +303,7 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: list, outgoing_endp """ Find information flows by matching incomgin to outgoing endpoints. Returns list of flows. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows_set = set() information_flows = dict() for o in outgoing_endpoints: diff --git a/technology_specific_extractors/spring_gateway/sgt_entry.py b/technology_specific_extractors/spring_gateway/sgt_entry.py index 0a8aee5..b2bd8d1 100644 --- a/technology_specific_extractors/spring_gateway/sgt_entry.py +++ b/technology_specific_extractors/spring_gateway/sgt_entry.py @@ -8,7 +8,7 @@ def detect_spring_cloud_gateway(dfd): """Detetcs Spring Cloud Gateway. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] external_components = dfd["external_components"] diff --git a/technology_specific_extractors/spring_oauth/soa_entry.py b/technology_specific_extractors/spring_oauth/soa_entry.py index 1d6480a..7b89b72 100644 --- a/technology_specific_extractors/spring_oauth/soa_entry.py +++ b/technology_specific_extractors/spring_oauth/soa_entry.py @@ -8,7 +8,7 @@ def detect_spring_oauth(dfd): """Detect Spring OAuth Server and connections to it. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] microservices = detect_authorization_server(microservices, dfd) diff --git a/technology_specific_extractors/zuul/zul_entry.py b/technology_specific_extractors/zuul/zul_entry.py index 9d7ac6b..f36d479 100644 --- a/technology_specific_extractors/zuul/zul_entry.py +++ b/technology_specific_extractors/zuul/zul_entry.py @@ -12,7 +12,7 @@ def detect_zuul(dfd): results = fi.search_keywords("@EnableZuulServer") new_results = fi.search_keywords("@EnableZuulProxy") - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows = dfd["information_flows"] external_components = dfd["external_components"] From 7e5d0f5b5b7ee203f26ea05931c086cd0dbd73d6 Mon Sep 17 00:00:00 2001 From: Alexander Bakhtin Date: Wed, 30 Oct 2024 23:03:38 +0200 Subject: [PATCH 10/10] add: container tech list similar to comm tech list --- core/technology_switch.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/core/technology_switch.py b/core/technology_switch.py index e246d85..b647eb0 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -9,18 +9,18 @@ import technology_specific_extractors.rabbitmq.rmq_entry as rmq import technology_specific_extractors.resttemplate.rst_entry as rst +CONTAINER_TECH_LIST = {"Maven": mvn, "Gradle": grd, "DockerCompose": dcm} COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, "Docker-Compose": dcm} def set_microservices(dfd): - """Calls get_microservices from correct container technology or returns existing list. + """Calls set_microservices from correct container technology or returns existing list. """ - mvn.set_microservices(dfd) - grd.set_microservices(dfd) - dcm.set_microservices(dfd) + for func in CONTAINER_TECH_LIST.values(): + func.set_microservices(dfd) def set_information_flows(dfd) -> dict: @@ -37,9 +37,7 @@ def detect_microservice(file_path: str, dfd) -> str: """Calls detect_microservices from correct microservice detection technology. """ - microservice = mvn.detect_microservice(file_path, dfd) - if not microservice: - microservice = grd.detect_microservice(file_path, dfd) - if not microservice: - microservice = dcm.detect_microservice(file_path, dfd) - return microservice + for func in CONTAINER_TECH_LIST.values(): + microservice = func.detect_microservice(file_path, dfd) + if microservice: + return microservice