Add a data provider for ISyntaxElements

src/devana/preprocessing/premade/components/flow/dataprovider.py

@@ -0,0 +1,51 @@
+from typing import List, Type, Optional, Callable, Iterable
+
+from devana.preprocessing.premade.components.executor.environment import Environment
+from devana.syntax_abstraction.organizers.sourcemodule import SourceModule
+from devana.syntax_abstraction.organizers.codecontainer import CodeContainer
+from devana.syntax_abstraction.syntax import ISyntaxElement
+from devana.preprocessing.preprocessor import ISource
+
+class SyntaxElementDataProvider(ISource):
+    """
+    Scans syntax elements across provided source modules and produces a list of
+    Environment.CallingData instances using custom factory functions.
+
+    If `deep_search` is True, method `feed()` will also look inside nested containers
+    (e.g. namespaces, classes) to find elements. If False, it only
+    checks the top-level elements that are directly inside each file.
+    """
+
+    def __init__(
+            self,
+            modules: List[SourceModule],
+            data_factories: List[Callable[[ISyntaxElement], Optional[Environment.CallingData]]],
+            deep_search: bool = True
+    ):
+        self._modules = modules
+        self._data_factories = data_factories
+        self._deep_search = deep_search
+
+    @classmethod
+    def get_produced_type(cls) -> Type:
+        return Environment.CallingData
+
+    def feed(self) -> List[Environment.CallingData]:
+        def get_syntax_elements(container: CodeContainer) -> Iterable[ISyntaxElement]:
+            """Yield all ISyntaxElement instances from container recursively."""
+            for content in container.content:
+                yield content
+                if isinstance(content, CodeContainer) and self._deep_search:
+                    yield from get_syntax_elements(content)
+
+        result = []
+        files = (file for module in self._modules for file in module.files)
+        elements = (element for file in files for element in get_syntax_elements(file))
+
+        for element in elements:
+            # call every factory with the current element
+            for factory in self._data_factories:
+                if data := factory(element):
+                    assert isinstance(data, Environment.CallingData), "Data is not an instance of Environment.CallingData"
+                    result.append(data)
+        return result

src/devana/preprocessing/premade/components/flow/sourcemerge.py

@@ -1,11 +1,11 @@
-from typing import List, Type, Any, TypeVar
+from typing import List, Type, Any
 from devana.preprocessing.preprocessor import ISource
 from devana.preprocessing.premade.components.executor.environment import Environment
-T = TypeVar('T')
 
 
 class SourceMergeCallingData(ISource):
     """Merges multiple sources into one."""
+
     def __init__(self, sources: List[ISource]):
         if len(sources) == 0:
             raise ValueError("No sources provided.")

src/devana/preprocessing/premade/components/parser/parser.py

@@ -7,9 +7,9 @@
 from devana.preprocessing.premade.components.parser.functionparser import FunctionParser
 from devana.preprocessing.premade.components.parser.typechecker import is_arguments_valid
 from devana.preprocessing.premade.components.executor.executable import CallFrame, Signature
-from devana.preprocessing.preprocessor import ISource
-from devana.syntax_abstraction.syntax import ISyntaxElement
 from devana.preprocessing.premade.components.executor.environment import Environment
+from devana.syntax_abstraction.syntax import ISyntaxElement
+from devana.preprocessing.preprocessor import ISource
 
 
 class Parser(ISource):
@@ -49,7 +49,6 @@ def _find_enum(cls, hint) -> List[Type[Enum]]:
     def get_produced_type(cls) -> Type:
         return Environment.CallingData
 
-
     def feed(self) -> List[Environment.CallingData[ISyntaxElement]]:
         result = []
         text_datas = self._extractor.extract()

tests/preprocessing/unit/premade/components/flow/source_files/elements/elements.hpp

@@ -0,0 +1,13 @@
+
+
+void p_function();
+
+struct Abc {
+    int a;
+};
+
+class p_Class;
+
+namespace test_namespace {
+    struct p_Struct{};
+}

tests/preprocessing/unit/premade/components/flow/test_dataprovider.py

@@ -0,0 +1,49 @@
+import unittest
+import os
+
+from typing import Optional
+from devana.preprocessing.premade.components.flow.dataprovider import SyntaxElementDataProvider
+from devana.preprocessing.premade.components.executor.environment import Environment, CallFrame
+from devana.preprocessing.premade.components.parser.parser import Signature
+from devana.syntax_abstraction.organizers.sourcemodule import SourceModule
+from devana.syntax_abstraction.syntax import ISyntaxElement
+
+
+def data_factory(element: ISyntaxElement) -> Optional[Environment.CallingData]:
+    name: Optional[str] = getattr(element, "name", None)
+    if name and name.startswith("p_"):
+        return Environment.CallingData(
+            arguments=CallFrame.Arguments(positional=[], named={}),
+            target=element,
+            signature=Signature(name="Parsable")
+        )
+
+
+class TestSyntaxElementDataProvider(unittest.TestCase):
+
+    def setUp(self):
+        self._module = SourceModule("Elements", os.path.dirname(__file__) + r"/source_files/elements")
+
+    def test_syntax_element_data_provider_with_deep_search(self):
+        provider = SyntaxElementDataProvider([self._module], [data_factory])
+        result = provider.feed()
+        self.assertEqual(len(result), 3)
+
+        for data in result:
+            self.assertTrue(isinstance(data, Environment.CallingData))
+            self.assertEqual(len(data.arguments.positional), 0)
+            self.assertEqual(len(data.arguments.named), 0)
+            self.assertEqual(data.signature.name, "Parsable")
+            self.assertEqual(data.signature.namespaces, [])
+
+    def test_syntax_element_data_provider_without_deep_search(self):
+        provider = SyntaxElementDataProvider([self._module], [data_factory], deep_search=False)
+        result = provider.feed()
+        self.assertEqual(len(result), 2)
+
+        for data in result:
+            self.assertTrue(isinstance(data, Environment.CallingData))
+            self.assertEqual(len(data.arguments.positional), 0)
+            self.assertEqual(len(data.arguments.named), 0)
+            self.assertEqual(data.signature.name, "Parsable")
+            self.assertEqual(data.signature.namespaces, [])

tests/preprocessing/unit/premade/components/parser/test_parser.py

@@ -1,5 +1,5 @@
 import unittest
-from typing import List
+from typing import List, Optional
 from enum import Enum, auto
 from devana.preprocessing.premade.components.parser.parser import Parser, Signature
 from devana.preprocessing.premade.components.parser.extractor import IExtractor, ExtractedFunction