improve regex readthedocs. Asserts about has package built

src/somef/header_analysis.py

@@ -109,7 +109,9 @@ def extract_header_content(text):
     df = pd.concat(dfs, ignore_index=True)
     # for i, j in zip(header, content):
     #     df = df.append({'Header': i, 'Content': j, 'ParentHeader': parent_headers[i]}, ignore_index=True)
-    df['Content'].replace('', np.nan, inplace=True)
+    # df['Content'].replace('', np.nan, inplace=True)
+    df['Content'] = df['Content'].replace('', np.nan)
+
     df.dropna(subset=['Content'], inplace=True)
     return df, none_header_content
 
@@ -221,13 +223,15 @@ def extract_categories(repo_data, repository_metadata: Result):
                 data.at[i, 'Group'] = data['GroupParent'][i]
         data = data.drop(columns=['GroupParent'])
         if len(data['Group'].iloc[0]) == 0:
-            data['Group'].iloc[0] = ['unknown']
+            # data['Group'].iloc[0] = ['unknown']
+            data.loc[0, 'Group'] = ['unknown']
         groups = data.apply(lambda x: pd.Series(x['Group']), axis=1).stack().reset_index(level=1, drop=True)
 
         groups.name = 'Group'
         data = data.drop('Group', axis=1).join(groups)
         if data['Group'].iloc[0] == 'unknown':
-            data['Group'].iloc[0] = np.NaN
+            # data['Group'].iloc[0] = np.NaN
+            data.loc[0, 'Group'] = np.nan
 
         # to json
         group = data.loc[(data['Group'] != 'None') & pd.notna(data['Group'])]

src/somef/parser/bower_parser.py

@@ -24,7 +24,8 @@ def parse_bower_json_file(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": "bower.json",
+                    # "value": "bower.json",
+                    "value": source,
                     "type": constants.URL,
                 },
                 1,

src/somef/parser/cabal_parser.py

@@ -24,7 +24,8 @@ def parse_cabal_file(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": Path(file_path).name,
+                    # "value": Path(file_path).name,
+                    "value": source,
                     "type": constants.URL,
                 },
                 1,

src/somef/parser/composer_parser.py

@@ -25,7 +25,8 @@ def parse_composer_json(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": "composer.json",
+                    # "value": "composer.json",
+                    "value": source,
                     "type": constants.URL,
                 },
                 1,

src/somef/parser/description_parser.py

@@ -24,7 +24,8 @@ def parse_description_file(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": "DESCRIPTION",
+                    # "value": "DESCRIPTION",
+                    "value": source,
                     "type": constants.URL,
                     "source": source
                 },

src/somef/parser/gemspec_parser.py

@@ -27,7 +27,8 @@ def parse_gemspec_file(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": Path(file_path).name,
+                    # "value": Path(file_path).name,
+                    "value": source,
                     "type": constants.URL,
                 },
                 1,

src/somef/parser/package_json_parser.py

@@ -188,7 +188,8 @@ def parse_package_json_file(file_path, metadata_result: Result, source):
             metadata_result.add_result(
                 constants.CAT_HAS_PACKAGE_FILE,
                 {
-                    "value": "package.json",
+                    # "value": "package.json",
+                    "value": source,
                     "type": constants.URL,
                 },
                 1,

src/somef/parser/pom_xml_parser.py

@@ -233,7 +233,8 @@ def parse_pom_file(file_path, metadata_result: Result, source):
         metadata_result.add_result(
             constants.CAT_HAS_PACKAGE_FILE,
             {
-                "value": "pom.xml",
+                # "value": "pom.xml",
+                "value": source,
                 "type": constants.URL
             },
             1,

src/somef/parser/toml_parser.py

@@ -45,7 +45,8 @@ def parse_toml_file(file_path, metadata_result: Result, source):
         metadata_result.add_result(
             constants.CAT_HAS_PACKAGE_FILE,
             {
-                "value": display_name,
+                # "value": display_name,
+                "value": source,
                 "type": constants.URL,
             },
             1,

src/somef/regular_expressions.py

@@ -72,7 +72,7 @@ def extract_title_old(unfiltered_text):
     # header declared with ====
     title = ""
     if len(underline_header) != 0:
-        title = re.split('.+[=]+[\n]+', unfiltered_text)[0].strip()
+        title = re.split(r'.+[=]+[\n]+', unfiltered_text)[0].strip()
     else:
         # The first occurrence is assumed to be the title.
         title = re.findall(r'# .+', unfiltered_text)[0]
@@ -81,7 +81,7 @@ def extract_title_old(unfiltered_text):
             title = title[1:].strip()
     # Remove other markup (links, etc.)
     if "[!" in title:
-        title = re.split('\[\!', title)[0].strip()
+        title = re.split(r'\[\!', title)[0].strip()
     return title
 
 
@@ -689,6 +689,7 @@ def extract_readthedocs_badgeds(readme_text, repository_metadata: Result, source
 
     # RST
     for match in re.findall(constants.REGEXP_READTHEDOCS_RST, readme_text):
+        print(match)
         if isinstance(match, tuple):
             urls.update([u for u in match if u])
         elif match:
@@ -702,15 +703,21 @@ def extract_readthedocs_badgeds(readme_text, repository_metadata: Result, source
             urls.add(match)
 
     # HTML
-    for match in re.findall(constants.REGEXP_READTHEDOCS_HTML, readme_text):
+    pattern_html = re.compile(constants.REGEXP_READTHEDOCS_HTML, flags=re.VERBOSE |re.DOTALL | re.IGNORECASE)
+    for match in pattern_html.findall(readme_text):
         if isinstance(match, tuple):
             urls.update([u for u in match if u])
         elif match:
             urls.add(match)
 
     for url in urls:
+        if "pypi.org/project" in url: 
+            category = constants.CAT_PACKAGE_DISTRIBUTION 
+        else: 
+            category = constants.CAT_DOCUMENTATION
+
         repository_metadata.add_result(
-            constants.CAT_DOCUMENTATION,
+            category,
             {
                 constants.PROP_TYPE: constants.URL,
                 constants.PROP_VALUE: url
@@ -722,28 +729,28 @@ def extract_readthedocs_badgeds(readme_text, repository_metadata: Result, source
 
     return repository_metadata
 
-def extract_package_manager_badgeds(readme_text, repository_metadata: Result, source) -> Result:
-    """
-    Function that takes the text of a readme file and searches if there are package manager badges.
-    Parameters
-    ----------
-    @param readme_text: Text of the readme
-    @param repository_metadata: Result with all the findings in the repo
-    @param source: source file on top of which the extraction is performed (provenance)
-    Returns
-    -------
-    @returns Result with the package badges found
-    """
-    package_manager_badges = re.findall(constants.REGEXP_READTHEDOCS_BADGES, readme_text, re.DOTALL)
-    for package in package_manager_badges:
-        repository_metadata.add_result(constants.CAT_DOCUMENTATION,
-                                       {
-                                           constants.PROP_TYPE: constants.URL,
-                                           constants.PROP_VALUE: package
-                                       }, 1, constants.TECHNIQUE_REGULAR_EXPRESSION, source)
+# def extract_package_manager_badgeds(readme_text, repository_metadata: Result, source) -> Result:
+#     """
+#     Function that takes the text of a readme file and searches if there are package manager badges.
+#     Parameters
+#     ----------
+#     @param readme_text: Text of the readme
+#     @param repository_metadata: Result with all the findings in the repo
+#     @param source: source file on top of which the extraction is performed (provenance)
+#     Returns
+#     -------
+#     @returns Result with the package badges found
+#     """
+#     package_manager_badges = re.findall(constants.REGEXP_READTHEDOCS_BADGES, readme_text, re.DOTALL)
+#     for package in package_manager_badges:
+#         repository_metadata.add_result(constants.CAT_DOCUMENTATION,
+#                                        {
+#                                            constants.PROP_TYPE: constants.URL,
+#                                            constants.PROP_VALUE: package
+#                                        }, 1, constants.TECHNIQUE_REGULAR_EXPRESSION, source)
 
 
-    return repository_metadata
+#     return repository_metadata
 
 
 def extract_swh_badges(readme_text, repository_metadata: Result, source) -> Result:

src/somef/test/test_bower_parser.py

@@ -16,10 +16,12 @@ def test_parse_bower_json(self):
 
         result = Result()
 
-        metadata_result = parse_bower_json_file(bower_file_path, result, "https://example.org/bower.json")
+        # metadata_result = parse_bower_json_file(bower_file_path, result, "https://example.org/bower.json")
+        metadata_result = parse_bower_json_file(bower_file_path, result, bower_file_path)
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "bower.json")
+        # self.assertEqual(package_results[0]["result"]["value"], "bower.json")
+        self.assertEqual(package_results[0]["result"]["value"], bower_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         name_results = metadata_result.results.get(constants.CAT_NAME, [])
@@ -70,10 +72,12 @@ def test_parse_2_bower_json(self):
 
         result = Result()
 
-        metadata_result = parse_bower_json_file(bower_file_path, result, "https://example.org/bower.json")
+        # metadata_result = parse_bower_json_file(bower_file_path, result, "https://example.org/bower.json")
+        metadata_result = parse_bower_json_file(bower_file_path, result, bower_file_path)
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "bower.json")
+        # self.assertEqual(package_results[0]["result"]["value"], "bower.json")
+        self.assertEqual(package_results[0]["result"]["value"], bower_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         name_results = metadata_result.results.get(constants.CAT_NAME, [])

src/somef/test/test_cabal_parser.py

@@ -15,11 +15,12 @@ def test_parse_cabal(self):
         cabal_file_path = test_data_repositories + os.path.sep + "unused" + os.path.sep + "unused.cabal"
         result = Result()
 
-        metadata_result = parse_cabal_file(cabal_file_path, result, "https://example.org/unused.cabal")
-
+        # metadata_result = parse_cabal_file(cabal_file_path, result, "https://example.org/unused.cabal")
+        metadata_result = parse_cabal_file(cabal_file_path, result, cabal_file_path)
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "unused.cabal")
+        # self.assertEqual(package_results[0]["result"]["value"], "unused.cabal")
+        self.assertEqual(package_results[0]["result"]["value"], cabal_file_path)  
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         id_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])
@@ -56,11 +57,12 @@ def test_parse_2_cabal(self):
         cabal_file_path = test_data_repositories + os.path.sep + "haskell" + os.path.sep + "cabal.cabal"
         result = Result()
 
-        metadata_result = parse_cabal_file(cabal_file_path, result, "https://example.org/cabal.cabal")
-
+        # metadata_result = parse_cabal_file(cabal_file_path, result, "https://example.org/cabal.cabal")
+        metadata_result = parse_cabal_file(cabal_file_path, result, cabal_file_path)
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "cabal.cabal")
+        # self.assertEqual(package_results[0]["result"]["value"], "cabal.cabal")
+        self.assertEqual(package_results[0]["result"]["value"], cabal_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
         description_results = metadata_result.results.get(constants.CAT_DESCRIPTION, [])
         self.assertTrue(len(description_results) > 0, "No description found")

src/somef/test/test_description_parser.py

@@ -15,7 +15,9 @@ def test_description(self):
         description_file_path = test_data_repositories + os.path.sep + "tidyverse" + os.path.sep + "DESCRIPTION"
         result = Result()
 
-        metadata_result = parse_description_file(description_file_path, result, "https://example.org/DESCRIPTION")
+        # metadata_result = parse_description_file(description_file_path, result, "https://example.org/DESCRIPTION")
+
+        metadata_result = parse_description_file(description_file_path, result, description_file_path)
 
         # print(metadata_result.results)
 
@@ -28,7 +30,9 @@ def test_description(self):
 
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "DESCRIPTION")
+        # self.assertEqual(package_results[0]["result"]["value"], "DESCRIPTION")
+        # self.assertEqual(package_results[0]["result"]["value"], "https://example.org/DESCRIPTION")
+        self.assertEqual(package_results[0]["result"]["value"], description_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         id_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])
@@ -49,17 +53,18 @@ def test_description(self):
 
     def test_description_2(self):
         description_file_path = test_data_repositories + os.path.sep + "ggplot2" + os.path.sep + "DESCRIPTION"
+
         result = Result()
 
-        metadata_result = parse_description_file(description_file_path, result, "https://example.org/DESCRIPTION")
-
+        # metadata_result = parse_description_file(description_file_path, result, "https://example.org/DESCRIPTION")
+        metadata_result = parse_description_file(description_file_path, result, description_file_path)
         authors_results = metadata_result.results.get(constants.CAT_AUTHORS, [])
         self.assertTrue(len(authors_results) == 11, "Expected 11 authors")
         self.assertEqual(authors_results[1]["result"]["value"],"Winston Chang","Second author name mismatch")
 
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "DESCRIPTION")
+        self.assertEqual(package_results[0]["result"]["value"], description_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         id_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])

src/somef/test/test_gemspec_parser.py

@@ -15,14 +15,16 @@ def test_parse_gemspec(self):
         gemspec_file_path = test_data_repositories + os.path.sep + "bootstrap-datepicker-rails" + os.path.sep + "bootstrap-datepicker-rails.gemspec"
         result = Result()
 
-        metadata_result = parse_gemspec_file(gemspec_file_path, result, "https://example.org/bootstrap-datepicker-rails.gemspec")
+        # metadata_result = parse_gemspec_file(gemspec_file_path, result, "https://example.org/bootstrap-datepicker-rails.gemspec")
+        metadata_result = parse_gemspec_file(gemspec_file_path, result, gemspec_file_path)
 
         authors_results = metadata_result.results.get(constants.CAT_AUTHORS, [])
         self.assertTrue(len(authors_results) == 2, "Expected two authors")
 
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "bootstrap-datepicker-rails.gemspec")
+        # self.assertEqual(package_results[0]["result"]["value"], "bootstrap-datepicker-rails.gemspec")
+        self.assertEqual(package_results[0]["result"]["value"], gemspec_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         id_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])

src/somef/test/test_package_json_parser.py

@@ -16,8 +16,8 @@ def test_parse_package_json_file(self):
         package_file_path = test_data_path + "package_neors.json"
         result = Result()
 
-        metadata_result = parse_package_json_file(package_file_path, result, "http://example.com/package_neors.json")
-
+        # metadata_result = parse_package_json_file(package_file_path, result, "http://example.com/package_neors.json")
+        metadata_result = parse_package_json_file(package_file_path, result, package_file_path)
         package_id_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])
         self.assertTrue(len(package_id_results) > 0, "No package ID found")
         self.assertEqual(package_id_results[0]["result"]["value"], "jsonlab")
@@ -58,7 +58,9 @@ def test_parse_package_json_file(self):
 
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "package.json")
+        # self.assertEqual(package_results[0]["result"]["value"], "package.json")
+        # self.assertEqual(package_results[0]["result"]["value"], "http://example.com/package_neors.json")
+        self.assertEqual(package_results[0]["result"]["value"],package_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         keywords_results = metadata_result.results.get(constants.CAT_KEYWORDS, [])

src/somef/test/test_pom_parser.py

@@ -17,8 +17,8 @@ def test_parse_pom_file(self):
         pom_xml_parser.processed_pom = False
         pom_file_path = test_data_repositories + os.path.sep + "Widoco" + os.path.sep + "pom.xml"
         result = Result()
-    
-        metadata_result = parse_pom_file(pom_file_path, result, "https://example.org/pom.xml")
+
+        metadata_result = parse_pom_file(pom_file_path, result, pom_file_path)
 
         identifier_results = metadata_result.results.get(constants.CAT_PACKAGE_ID, [])
         self.assertTrue(len(identifier_results) > 0, "No identifier found")
@@ -34,7 +34,8 @@ def test_parse_pom_file(self):
 
         package_results = metadata_result.results.get(constants.CAT_HAS_PACKAGE_FILE, [])
         self.assertTrue(len(package_results) > 0, "No package file info found")
-        self.assertEqual(package_results[0]["result"]["value"], "pom.xml")
+        # self.assertEqual(package_results[0]["result"]["value"], "pom.xml")
+        self.assertEqual(package_results[0]["result"]["value"], pom_file_path)
         self.assertEqual(package_results[0]["result"]["type"], constants.URL)
 
         requirements_results = metadata_result.results.get(constants.CAT_REQUIREMENTS, [])