fix:1.修复表格内公式没有被$$包裹渲染异常 2.去掉原生HTML具有hidden属性的隐藏标签

llm_web_kit/extractor/config.py

@@ -3,6 +3,7 @@
     {'url': '*', 'tag': '//div[starts-with(@class, "advert") or starts-with(@name, "advert") or starts-with(@id, "advert")]'},
     {'url': '*', 'tag': '//div[contains(@style, "display: none")]'},
     {'url': '*', 'tag': '//div[contains(@style, "display:none")]'},
+    {'url': '*', 'tag': '//*[@hidden and not(@hidden="false")]'},
     {'url': 'stackexchange.com', 'tag': '//*[contains(@class, "d-none")]'},  # 任意标签，class包含d-none，限制在stackexchange.com网站
     {'url': 'mathoverflow.net', 'tag': '//*[contains(@class, "d-none")]'},  # 任意标签，class包含d-none，限制在mathoverflow.net网站
     {'url': 'blog.csdn.net', 'tag': '//span[contains(@class, "katex-html")]'},  # 仅针对 blog.csdn.net 域名，删除所有 class 包含 katex-html 的 <span> 标签及其内容（用于移除数学公式渲染的 HTML 部分）

llm_web_kit/extractor/html/recognizer/cc_math/tag_math.py

@@ -24,11 +24,12 @@ def modify_tree(cm: CCMATH, math_render: str, o_html: str, node: HtmlElement, pa
         if len(annotation_tags) > 0:
             annotation_tag = annotation_tags[0]
             text = annotation_tag.text
-            style_value = parent.get('style')
-            if style_value:
-                normalized_style_value = style_value.lower().strip().replace(' ', '').replace(';', '')
-                if 'display: none' in normalized_style_value:
-                    parent.style = ''
+            if parent:
+                style_value = parent.get('style')
+                if style_value:
+                    normalized_style_value = style_value.lower().strip().replace(' ', '').replace(';', '')
+                    if 'display: none' in normalized_style_value:
+                        parent.style = ''
             text = cm.wrap_math_md(text)
             if text:
                 new_span = build_cc_element(html_tag_name=new_tag, text=text, tail=text_strip(node.tail), type=math_type, by=math_render, html=o_html)

llm_web_kit/extractor/html/recognizer/table.py

@@ -1,3 +1,4 @@
+import copy
 import re
 from typing import Any, List, Tuple
 
@@ -212,16 +213,16 @@ def __get_table_type(self, child: HtmlElement) -> str:
 
     def __check_table_include_math_code(self, raw_html: HtmlElement):
         """检查table中的内容，包括普通文本、数学公式和代码."""
-        math_raw_html = self._element_to_html(raw_html)
-        math_html = raw_html
+        tem_raw_html = copy.deepcopy(raw_html)
+        math_raw_html = self._element_to_html(tem_raw_html)
         math_res_parts = self.math_recognizer.recognize(
             base_url='',
-            main_html_lst=[(math_html, math_html)],
+            main_html_lst=[(tem_raw_html, tem_raw_html)],
             raw_html=math_raw_html
         )
         result = []
         if not math_res_parts:
-            if raw_html.tag == 'br' or raw_html.xpath('.//br'):
+            if tem_raw_html.tag == 'br' or tem_raw_html.xpath('.//br'):
                 result.append("\n\n")
         for math_item in math_res_parts:
             ele_item = math_item[0]

tests/llm_web_kit/extractor/html/recognizer/test_table.py

@@ -234,3 +234,11 @@ def test_nested_table4(self):
         assert len(parts) == 4
         content = parts[2][0].text_content()
         assert '<table><tr><td>Molecular line emissions from pre main sequence objects\n\nSaraceno, P. ; Benedettini, M. ; Caux, E. ; Ceccarelli, M. C. ; Clegg, P. E. ; Correia, J. C. ; di Giorgio, A. M. ; Giannini, T. ; Griffin, M. J. ; Leeks, S. J. ; Liseau, R. ; Lorenzetti, D. ; Molinari, S. ; Nisini, B. ; Smith, H. ; Spinoglio, L. ; Tomassi, E. and White, G. J. (1997). Molecular line emissions from pre main sequence objects. In: The first ISO workshop on Analytical Spectroscopy , 6-8 October 1997, Madrid, Spain, p. 291.\n\nFull text available as:<table><tr><td><table><tr><td>Preview</td></tr></table></td><td>PDF (Version of Record) - Requires a PDF viewer such asGSview ,Xpdf orAdobe Acrobat Reader\n\nDownload (239Kb)</td></tr></table><table><tr><th>URL:</th><td>http://cdsads.u-strasbg.fr/abs/1997ESASP.419..291S</td></tr><tr><th>Google Scholar:</th><td>Look up in Google Scholar</td></tr></table>Abstract\n\nWe present some preliminary results obtained with the LWS G.T. programme on the study of young objects driving molecular outflows. In particular, we discuss the importance of molecular emission in these sources and address the role of the H<sub>2</sub>0 cooling.<table><tr><th>Item Type:</th><td>Conference Item</td></tr><tr><th>Copyright Holders:</th><td>1997 European Space Agency</td></tr><tr><th>Extra Information:</th><td>Proceedings of the first ISO workshop on Analytical Spectroscopy, Madrid, Spain, 6-8 October 1997. Editors: A.M. Heras, K. Leech, N. R. Trams, and Michael Perry. Noordwijk, The Netherlands : ESA Publications Division, c1997. (ESA SP-419), 1997., pp.291-292</td></tr><tr><th>Academic Unit/Department:</th><td>Science > Physical Sciences</td></tr><tr><th>Interdisciplinary Research Centre:</th><td>Centre for Earth, Planetary, Space and Astronomical Research (CEPSAR)</td></tr><tr><th>Item ID:</th><td>32696</td></tr><tr><th>Depositing User:</th><td>Glenn White</td></tr>' in content
+
+    def test_table_has_formula(self):
+        """表格含有公式."""
+        raw_html_path = base_dir.joinpath('assets/recognizer/table_has_formula.html')
+        base_url = 'https://en.m.wikipedia.org/wiki/Variance'
+        raw_html = raw_html_path.read_text(encoding='utf-8')
+        parts = self.rec.recognize(base_url, [(html_to_element(raw_html), html_to_element(raw_html))], raw_html)
+        assert parts[5][0].text_content() == '<table><thead><tr><th>Statistic</th><th># (units)</th></tr></thead><tbody><tr><td>Number of patients</td><td>11,000</td></tr><tr><td>Number of labeled beats</td><td>2,774,054,987</td></tr><tr><td>Sample rate</td><td>250Hz</td></tr><tr><td>Segment size</td><td>$2^{20}+1$= 1,048,577</td></tr><tr><td>Total number of segments</td><td>541,794 (not all patients have enough for 50 segments)</td></tr></tbody></table>'

tests/llm_web_kit/extractor/html/recognizer/test_text.py

@@ -503,6 +503,30 @@ def test_para_has_none(self):
         content_md = result.get_content_list().to_mm_md()
         assert content_md
 
+    def test_clean_invisible_elements(self):
+        """
+        清理隐藏标签
+        Returns:
+
+        """
+        chain = ExtractSimpleFactory.create(load_pipe_tpl('noclip_html_test'))
+        self.assertIsNotNone(chain)
+        test_data = {
+            'track_id': 'text_md',
+            'dataset_name': 'text_md',
+            'url': 'https://br.wikipedia.org/wiki/Faustina_an_Hena%C3%B1',
+            'data_source_category': 'HTML',
+            'path': 'clean_invisible_elements.html',
+            'main_path': 'clean_invisible_elements.html',
+            'file_bytes': 1000,
+            'meta_info': {'input_datetime': '2020-01-01 00:00:00'},
+            'language': 'en'
+        }
+        input_data = DataJson(test_data)
+        result = chain.extract(input_data)
+        content_md = result.get_content_list().to_mm_md()
+        assert "Choosing a selection results in a full page refresh." not in content_md
+
     def test_empty_string_fix(self):
         """
         测试修复字符串索引越界问题 - 当文本处理中出现空字符串时不应抛出IndexError