From a24c353e0b46063758c984bb51976f94439cd441 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 29 Dec 2025 14:54:51 +0800 Subject: [PATCH 1/4] Added support for .tasty and .scala relation #1871 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/d2d.py | 4 +-- scanpipe/pipes/jvm.py | 2 +- scanpipe/tests/pipes/test_d2d.py | 50 +++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index a428400129..c8e3b64294 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -239,8 +239,8 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None): if logger: logger( - f"Mapping {to_resource_count:,d} .class resources to " - f"{from_resource_count:,d} {jvm_lang.source_extensions}" + f"Mapping {to_resource_count:,d} .class (or other deployed file) " + f"resources to {from_resource_count:,d} {jvm_lang.source_extensions}" ) # build an index using from-side fully qualified class file names diff --git a/scanpipe/pipes/jvm.py b/scanpipe/pipes/jvm.py index b071d9dcd4..ab4dcfd10f 100644 --- a/scanpipe/pipes/jvm.py +++ b/scanpipe/pipes/jvm.py @@ -192,7 +192,7 @@ class JavaLanguage(JvmLanguage): class ScalaLanguage(JvmLanguage): name = "scala" source_extensions = (".scala",) - binary_extensions = (".class",) + binary_extensions = (".class", ".tasty") source_package_attribute_name = "scala_package" package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") binary_map_type = "scala_to_class" diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 4d8433498e..49d5b09067 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -372,7 +372,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 3 .class resources to 2 ('.java',)" + expected = "Mapping 3 .class (or other deployed file) resources to 2 ('.java',)" self.assertIn(expected, buffer.getvalue()) self.assertEqual(2, self.project1.codebaserelations.count()) @@ -432,9 +432,47 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self): d2d.map_jvm_to_class( self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 1 .class resources to 1 ('.java',)" + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.java',)" self.assertIn(expected, buffer.getvalue()) + def test_scanpipe_pipes_d2d_map_scala_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/tastyquery/Annotations.scala", + extra_data={"scala_package": "tastyquery"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.tasty", + ) + + to2 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage + ) + + expected = ( + "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(2, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("scala_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1) + self.assertEqual("scala_to_class", r2.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r2.extra_data) + def test_scanpipe_pipes_d2d_map_grammar_to_class(self): from1 = make_resource_file( self.project1, @@ -452,7 +490,9 @@ def test_scanpipe_pipes_d2d_map_grammar_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.GrammarLanguage ) - expected = "Mapping 1 .class resources to 1 ('.g', '.g4')" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.g', '.g4')" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -480,7 +520,9 @@ def test_scanpipe_pipes_d2d_map_xtend_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.XtendLanguage ) - expected = "Mapping 1 .class resources to 1 ('.xtend',)" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.xtend',)" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) From 57a7d8795d163c3fa7c9fbf60ac4db2d78b343ab Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 30 Dec 2025 13:34:52 +0800 Subject: [PATCH 2/4] Added Groovy support #1871 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/deploy_to_develop.py | 25 ++++++ scanpipe/pipes/d2d_config.py | 8 ++ scanpipe/pipes/jvm.py | 9 ++ scanpipe/tests/pipes/test_d2d.py | 106 +++++++++++++++--------- 4 files changed, 110 insertions(+), 38 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 9d60ac1dc4..4755ed993c 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -83,6 +83,9 @@ def steps(cls): cls.find_grammar_packages, cls.map_grammar_to_class, cls.map_jar_to_grammar_source, + cls.find_groovy_packages, + cls.map_groovy_to_class, + cls.map_jar_to_groovy_source, cls.find_xtend_packages, cls.map_xtend_to_class, cls.map_javascript, @@ -262,6 +265,28 @@ def map_jar_to_grammar_source(self): project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log ) + @optional_step("Groovy") + def find_groovy_packages(self): + """Find the java package of the .java source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_groovy_to_class(self): + """Map a .class compiled file to its .java source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_jar_to_groovy_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Xtend") def find_xtend_packages(self): """Find the java package of the xtend source files.""" diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index d965feae84..05ea09ca44 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -112,6 +112,14 @@ class EcosystemConfig: "*kotlin-project-structure-metadata.json", ], ), + "Groovy": EcosystemConfig( + ecosystem_option="Groovy", + matchable_package_extensions=[".jar", ".war"], + matchable_resource_extensions=[".class"], + deployed_resource_path_exclusions=[ + "*META-INF/*", + ], + ), "JavaScript": EcosystemConfig( ecosystem_option="JavaScript", matchable_resource_extensions=[ diff --git a/scanpipe/pipes/jvm.py b/scanpipe/pipes/jvm.py index ab4dcfd10f..bd6ee1da0d 100644 --- a/scanpipe/pipes/jvm.py +++ b/scanpipe/pipes/jvm.py @@ -198,6 +198,15 @@ class ScalaLanguage(JvmLanguage): binary_map_type = "scala_to_class" +class GroovyLanguage(JvmLanguage): + name = "groovy" + source_extensions = (".groovy",) + binary_extensions = (".class",) + source_package_attribute_name = "groovy_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "groovy_to_class" + + class KotlinLanguage(JvmLanguage): name = "kotlin" source_extensions = (".kt", ".kts") diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 49d5b09067..74c98909b6 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -435,44 +435,6 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self): expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.java',)" self.assertIn(expected, buffer.getvalue()) - def test_scanpipe_pipes_d2d_map_scala_to_class(self): - from1 = make_resource_file( - self.project1, - path="from/tastyquery/Annotations.scala", - extra_data={"scala_package": "tastyquery"}, - ) - - to1 = make_resource_file( - self.project1, - path="to/tastyquery/Annotations.tasty", - ) - - to2 = make_resource_file( - self.project1, - path="to/tastyquery/Annotations.class", - ) - - buffer = io.StringIO() - d2d.map_jvm_to_class( - self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage - ) - - expected = ( - "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)" - ) - self.assertIn(expected, buffer.getvalue()) - self.assertEqual(2, self.project1.codebaserelations.count()) - - r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) - self.assertEqual("scala_to_class", r1.map_type) - expected = {"from_source_root": "from/"} - self.assertEqual(expected, r1.extra_data) - - r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1) - self.assertEqual("scala_to_class", r2.map_type) - expected = {"from_source_root": "from/"} - self.assertEqual(expected, r2.extra_data) - def test_scanpipe_pipes_d2d_map_grammar_to_class(self): from1 = make_resource_file( self.project1, @@ -611,6 +573,74 @@ def test_scanpipe_pipes_d2d_map_jar_to_java_source(self): self.assertEqual(from2, relation.from_resource) self.assertEqual(to_jar, relation.to_resource) + def test_scanpipe_pipes_d2d_map_groovy_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.groovy", + extra_data={"groovy_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.GroovyLanguage + ) + + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.groovy',)" + ) + print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("groovy_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + + def test_scanpipe_pipes_d2d_map_scala_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/tastyquery/Annotations.scala", + extra_data={"scala_package": "tastyquery"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.tasty", + ) + + to2 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage + ) + + expected = ( + "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(2, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("scala_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1) + self.assertEqual("scala_to_class", r2.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r2.extra_data) + def test_scanpipe_pipes_d2d_map_jar_to_scala_source(self): from1 = make_resource_file( self.project1, From 8be6c8a0fecbd57da0279981167f904de539554f Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 30 Dec 2025 14:53:46 +0800 Subject: [PATCH 3/4] Added support for aspectj and clogure #1871 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/deploy_to_develop.py | 57 ++++++++++++++++++++++--- scanpipe/pipes/jvm.py | 18 ++++++++ scanpipe/tests/pipes/test_d2d.py | 53 +++++++++++++++++++++++ 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 4755ed993c..fa481fa95f 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -86,6 +86,12 @@ def steps(cls): cls.find_groovy_packages, cls.map_groovy_to_class, cls.map_jar_to_groovy_source, + cls.find_aspectj_packages, + cls.map_aspectj_to_class, + cls.map_jar_to_aspectj_source, + cls.find_clojure_packages, + cls.map_clojure_to_class, + cls.map_jar_to_clojure_source, cls.find_xtend_packages, cls.map_xtend_to_class, cls.map_javascript, @@ -211,7 +217,7 @@ def find_scala_packages(self): @optional_step("Scala") def map_scala_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its .scala source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log ) @@ -225,14 +231,14 @@ def map_jar_to_scala_source(self): @optional_step("Kotlin") def find_kotlin_packages(self): - """Find the java package of the .java source files.""" + """Find the java package of the kotlin source files.""" d2d.find_jvm_packages( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @optional_step("Kotlin") def map_kotlin_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its kotlin source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @@ -267,14 +273,14 @@ def map_jar_to_grammar_source(self): @optional_step("Groovy") def find_groovy_packages(self): - """Find the java package of the .java source files.""" + """Find the package of the .groovy source files.""" d2d.find_jvm_packages( project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log ) @optional_step("Groovy") def map_groovy_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its .groovy source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log ) @@ -286,6 +292,47 @@ def map_jar_to_groovy_source(self): project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log ) + @optional_step("AspectJ") + def find_aspectj_packages(self): + """Find the package of the .aj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_aspectj_to_class(self): + """Map a .class compiled file to its .aj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_jar_to_aspectj_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("Clojure") + def find_clojure_packages(self): + """Find the package of the .clj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_clojure_to_class(self): + """Map a .class compiled file to its .clj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_jar_to_clojure_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) @optional_step("Xtend") def find_xtend_packages(self): diff --git a/scanpipe/pipes/jvm.py b/scanpipe/pipes/jvm.py index bd6ee1da0d..ae7b5b9203 100644 --- a/scanpipe/pipes/jvm.py +++ b/scanpipe/pipes/jvm.py @@ -207,6 +207,24 @@ class GroovyLanguage(JvmLanguage): binary_map_type = "groovy_to_class" +class AspectJLanguage(JvmLanguage): + name = "aspectj" + source_extensions = (".aj",) + binary_extensions = (".class",) + source_package_attribute_name = "aspectj_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "aspectj_to_class" + + +class ClojureLanguage(JvmLanguage): + name = "clojure" + source_extensions = (".clj",) + binary_extensions = (".class",) + source_package_attribute_name = "clojure_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "clojure_to_class" + + class KotlinLanguage(JvmLanguage): name = "kotlin" source_extensions = (".kt", ".kts") diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 74c98909b6..3114d2a920 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -602,6 +602,59 @@ def test_scanpipe_pipes_d2d_map_groovy_to_class(self): expected = {"from_source_root": "from/"} self.assertEqual(expected, r1.extra_data) + def test_scanpipe_pipes_d2d_map_aspectj_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.aj", + extra_data={"aspectj_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.AspectJLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.aj',)" + print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("aspectj_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_clojure_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.clj", + extra_data={"clojure_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ClojureLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.clj',)" + print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("clojure_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) def test_scanpipe_pipes_d2d_map_scala_to_class(self): from1 = make_resource_file( From 6c57d87be65fd495010a4ca6f76e1d00ee5684b5 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 30 Dec 2025 15:07:23 +0800 Subject: [PATCH 4/4] Remove test print statements #1871 Signed-off-by: Chin Yeung Li --- scanpipe/tests/pipes/test_d2d.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 3114d2a920..73db8c0c9e 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -593,7 +593,6 @@ def test_scanpipe_pipes_d2d_map_groovy_to_class(self): expected = ( "Mapping 1 .class (or other deployed file) resources to 1 ('.groovy',)" ) - print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -620,7 +619,6 @@ def test_scanpipe_pipes_d2d_map_aspectj_to_class(self): ) expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.aj',)" - print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -647,7 +645,6 @@ def test_scanpipe_pipes_d2d_map_clojure_to_class(self): ) expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.clj',)" - print("buffer.getvalue():", buffer.getvalue(), file=sys.stderr) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count())