diff --git a/.dockerignore b/.dockerignore index 4b0c03a40..36bf089a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -41,4 +41,4 @@ coverage/ # Docker .dockerignore -Dockerfile \ No newline at end of file +Dockerfile diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index c57baf24d..68f900727 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -32,4 +32,4 @@ jobs: uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 \ No newline at end of file + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f8a8dd1ec..51b706c20 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -26,4 +26,4 @@ jobs: # Run the mkdocs command using Poetry's environment - name: Deploy documentation run: | - poetry run mkdocs gh-deploy --force \ No newline at end of file + poetry run mkdocs gh-deploy --force diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..1f8b1e094 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,51 @@ +name: Lint + +permissions: + contents: read + +on: + push: + branches: [main] + paths: + - '**.py' + - 'pyproject.toml' + - '.github/workflows/lint.yml' + pull_request: + branches: [main] + paths: + - '**.py' + - 'pyproject.toml' + - '.github/workflows/lint.yml' + workflow_dispatch: + +jobs: + ruff: + name: Ruff Linting + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install dependencies + run: | + poetry install --no-interaction --no-ansi + + - name: Run ruff check + run: | + poetry run ruff check --output-format=github . + + - name: Run ruff format check + run: | + poetry run ruff format --check . diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b7e2242c5..09d2fd542 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -33,10 +33,10 @@ jobs: cd flowfile_frontend npm install npm run build:web - + # Create the static directory if it doesn't exist mkdir -p ../flowfile/flowfile/web/static - + # Copy the built files to the Python package cp -r build/renderer/* ../flowfile/flowfile/web/static/ echo "Contents of web/static directory:" @@ -81,4 +81,4 @@ jobs: with: skip-existing: true packages-dir: dist/ - verbose: true \ No newline at end of file + verbose: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index eb2c54991..2654432e9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,4 +112,4 @@ jobs: echo "GitHub Release created." echo "Tag: ${{ github.ref_name }}" echo "Release ID: ${{ steps.create_release.outputs.id }}" - echo "Assets have been uploaded to the GitHub Release." \ No newline at end of file + echo "Assets have been uploaded to the GitHub Release." diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee10edc9d..d96b00b1a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -52,7 +52,7 @@ jobs: cd flowfile_frontend npm install npm run build:web - + # Create the static directory if it doesn't exist mkdir -p ../flowfile/flowfile/web/static @@ -112,7 +112,7 @@ jobs: cd flowfile_frontend npm install npm run build:web - + # Create the static directory if it doesn't exist New-Item -ItemType Directory -Force -Path ../flowfile/flowfile/web/static | Out-Null @@ -283,4 +283,4 @@ jobs: shell: pwsh working-directory: flowfile_frontend run: | - npm run test \ No newline at end of file + npm run test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..6bc22c6d2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,39 @@ +# Pre-commit hooks configuration +# See https://pre-commit.com for more information + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version should match the version in pyproject.toml + rev: v0.8.6 + hooks: + # Run the linter + - id: ruff + args: [--fix] + types_or: [python, pyi] + # Run the formatter + - id: ruff-format + types_or: [python, pyi] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + # Identify invalid files + - id: check-ast + - id: check-yaml + - id: check-json + exclude: 'tsconfig.*\.json$' # Exclude TypeScript config files (they use JSONC with comments) + - id: check-toml + # Check for files that would conflict in case-insensitive filesystems + - id: check-case-conflict + # Check for merge conflicts + - id: check-merge-conflict + # Check for debugger imports + - id: debug-statements + # Make sure files end with newline + - id: end-of-file-fixer + # Trim trailing whitespace + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Check for large files + - id: check-added-large-files + args: ['--maxkb=1000'] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..508ca4ba1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,142 @@ +# Contributing to Flowfile + +Thank you for your interest in contributing to Flowfile! This guide will help you set up your development environment and understand our code quality standards. + +## Development Setup + +### Prerequisites + +- Python 3.10 or higher (but less than 3.14) +- [Poetry](https://python-poetry.org/docs/#installation) for dependency management +- Git + +### Initial Setup + +1. **Clone the repository** + ```bash + git clone https://github.com/Edwardvaneechoud/Flowfile.git + cd Flowfile + ``` + +2. **Install dependencies** + ```bash + poetry install + ``` + +3. **Install pre-commit hooks** (recommended) + ```bash + poetry run pre-commit install + ``` + + This will automatically run linting and formatting checks before each commit. + +## Code Quality + +### Linting with Ruff + +We use [Ruff](https://docs.astral.sh/ruff/) for linting and code formatting. Ruff is configured in `pyproject.toml`. + +**Run linting manually:** +```bash +# Check for linting issues +poetry run ruff check . + +# Auto-fix linting issues +poetry run ruff check --fix . + +# Check code formatting +poetry run ruff format --check . + +# Format code +poetry run ruff format . +``` + +**Configuration:** +- Target: Python 3.10+ +- Line length: 120 characters +- Rules: F (Pyflakes), E/W (pycodestyle), I (isort), UP (pyupgrade), B (flake8-bugbear) + +### Pre-commit Hooks + +Pre-commit hooks automatically run before each commit to ensure code quality. They will: + +1. **Ruff linting** - Check and auto-fix Python code issues +2. **Ruff formatting** - Format Python code consistently +3. **File checks** - Validate YAML, JSON, TOML, and Python syntax +4. **Trailing whitespace** - Remove unnecessary whitespace +5. **End of file** - Ensure files end with a newline +6. **Merge conflicts** - Detect merge conflict markers +7. **Large files** - Prevent committing large files (>1MB) + +**Skip pre-commit hooks** (not recommended): +```bash +git commit --no-verify -m "Your commit message" +``` + +**Run pre-commit manually on all files:** +```bash +poetry run pre-commit run --all-files +``` + +### Continuous Integration + +Our GitHub Actions workflows automatically run: + +- **Linting** (`lint.yml`) - Runs ruff check and format validation on all PRs +- **Tests** (`test-docker-auth.yml`, `e2e-tests.yml`) - Runs test suites +- **Documentation** (`documentation.yml`) - Builds and deploys docs + +All checks must pass before a PR can be merged. + +## Running Tests + +```bash +# Run all tests +poetry run pytest + +# Run tests for a specific module +poetry run pytest flowfile_core/tests/ +poetry run pytest flowfile_worker/tests/ + +# Run tests with coverage +poetry run pytest --cov=flowfile_core --cov=flowfile_worker +``` + +## Code Style Guidelines + +- Follow [PEP 8](https://pep8.org/) style guidelines (enforced by Ruff) +- Use type hints where appropriate +- Write descriptive variable and function names +- Keep functions focused and modular +- Add docstrings for public functions and classes +- Keep line length under 120 characters + +## Submitting Changes + +1. **Create a new branch** for your feature or fix: + ```bash + git checkout -b feature/your-feature-name + ``` + +2. **Make your changes** and ensure all tests pass + +3. **Commit your changes** (pre-commit hooks will run automatically): + ```bash + git add . + git commit -m "Add your descriptive commit message" + ``` + +4. **Push to your fork**: + ```bash + git push origin feature/your-feature-name + ``` + +5. **Create a Pull Request** on GitHub + +## Getting Help + +- Check the [documentation](https://edwardvaneechoud.github.io/Flowfile/) +- Open an issue on GitHub +- Read the [architecture documentation](docs/for-developers/architecture.md) + +Thank you for contributing to Flowfile! 🚀 diff --git a/README.md b/README.md index 05cb56a5b..392bd3340 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ For a deeper dive into the technical architecture, check out [this article](http #### 1. Desktop Application The desktop version offers the best experience with a native interface and integrated services. You can either: -**Option A: Download Pre-built Application** +**Option A: Download Pre-built Application** - Download the latest release from [GitHub Releases](https://github.com/Edwardvaneechoud/Flowfile/releases) - Run the installer for your platform (Windows, macOS, or Linux) > **Note:** You may see security warnings since the app isn't signed with a developer certificate yet. diff --git a/build_backends/build_backends/main.py b/build_backends/build_backends/main.py index d8ac6c21e..54ac460d5 100644 --- a/build_backends/build_backends/main.py +++ b/build_backends/build_backends/main.py @@ -50,7 +50,7 @@ def get_connectorx_metadata(): for dist_info in glob.glob(dist_info_pattern): metadata_locations.append(dist_info) - # Look for egg-info directories + # Look for egg-info directories egg_info_pattern = os.path.join(site_packages, 'connectorx*.egg-info') for egg_info in glob.glob(egg_info_pattern): metadata_locations.append(egg_info) @@ -126,7 +126,7 @@ def patched_version(distribution_name): """ # Collect minimal snowflake dependencies -snowflake_imports = collect_submodules('snowflake.connector', +snowflake_imports = collect_submodules('snowflake.connector', filter=lambda name: any(x in name for x in [ 'connection', 'errors', diff --git a/build_backends/build_backends/main_prd.py b/build_backends/build_backends/main_prd.py index c4f41522c..b9584e0d1 100644 --- a/build_backends/build_backends/main_prd.py +++ b/build_backends/build_backends/main_prd.py @@ -22,7 +22,7 @@ def wait_for_endpoint(url, timeout=60): def shutdown_service(): """Shutdown the service gracefully using the shutdown endpoint.""" try: - response = requests.post("http://0.0.0.0:63578/shutdown", headers={"accept": "application/json"}, data="") + requests.post("http://0.0.0.0:63578/shutdown", headers={"accept": "application/json"}, data="") print("Shutdown request sent, waiting for service to stop...") time.sleep(1) # Wait 10 seconds to ensure the service is fully stopped return True diff --git a/docker-compose.yml b/docker-compose.yml index 7686581fb..8452d4dc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,4 +88,4 @@ volumes: secrets: flowfile_master_key: - file: ./master_key.txt \ No newline at end of file + file: ./master_key.txt diff --git a/docs/MakeFile b/docs/MakeFile index 3d0ce63c6..8cf77a9cf 100644 --- a/docs/MakeFile +++ b/docs/MakeFile @@ -20,4 +20,4 @@ clean: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/for-developers/architecture.md b/docs/for-developers/architecture.md index 51556a41c..bc999dc27 100644 --- a/docs/for-developers/architecture.md +++ b/docs/for-developers/architecture.md @@ -239,4 +239,4 @@ This design enables Flowfile to: --- -*For a deep dive into the implementation details, see the [full technical article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c).* \ No newline at end of file +*For a deep dive into the implementation details, see the [full technical article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c).* diff --git a/docs/for-developers/creating-custom-nodes.md b/docs/for-developers/creating-custom-nodes.md index a9e8a218d..7b2b3179e 100644 --- a/docs/for-developers/creating-custom-nodes.md +++ b/docs/for-developers/creating-custom-nodes.md @@ -73,7 +73,7 @@ class GreetingNode(CustomNodeBase): node_category: str = "Text Processing" title: str = "Add Personal Greetings" intro: str = "Transform names into personalized greetings" - + settings_schema: GreetingSettings = GreetingSettings() def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: @@ -81,27 +81,27 @@ class GreetingNode(CustomNodeBase): name_col = self.settings_schema.main_config.name_column.value style = self.settings_schema.main_config.greeting_style.value custom = self.settings_schema.main_config.custom_message.value - + # Define greeting logic if style == "formal": greeting_expr = pl.concat_str([ - pl.lit("Hello, "), - pl.col(name_col), + pl.lit("Hello, "), + pl.col(name_col), pl.lit(f". {custom}") ]) elif style == "casual": greeting_expr = pl.concat_str([ - pl.lit("Hey "), - pl.col(name_col), + pl.lit("Hey "), + pl.col(name_col), pl.lit(f"! {custom}") ]) else: # enthusiastic greeting_expr = pl.concat_str([ - pl.lit("OMG HI "), - pl.col(name_col).str.to_uppercase(), + pl.lit("OMG HI "), + pl.col(name_col).str.to_uppercase(), pl.lit(f"!!! {custom} 🎉") ]) - + return input_df.with_columns([ greeting_expr.alias("greeting") ]) @@ -141,7 +141,7 @@ class MyCustomNode(CustomNodeBase): # 2. Settings Schema - The UI configuration settings_schema: MySettings = MySettings() - + # 3. Processing Logic - What the node actually does def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: # Your transformation logic here @@ -184,7 +184,7 @@ class MyNodeSettings(NodeSettings): input_column=ColumnSelector(...), operation_type=SingleSelect(...) ) - + advanced_options: Section = Section( title="Advanced Options", description="Fine-tune behavior", @@ -341,7 +341,7 @@ data_types=[Types.Numeric, Types.Date] # Numbers and dates only class DataQualityNode(CustomNodeBase): node_name: str = "Data Quality Checker" node_category: str = "Data Validation" - + settings_schema: DataQualitySettings = DataQualitySettings( validation_rules=Section( title="Validation Rules", @@ -366,7 +366,7 @@ class DataQualityNode(CustomNodeBase): def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: columns = self.settings_schema.validation_rules.columns_to_check.value threshold = self.settings_schema.validation_rules.null_threshold.value - + # Calculate quality metrics quality_checks = [] for col in columns: @@ -376,7 +376,7 @@ class DataQualityNode(CustomNodeBase): "null_percentage": null_pct, "quality_flag": "PASS" if null_pct <= threshold else "FAIL" }) - + # Add quality flags to original data result_df = input_df for check in quality_checks: @@ -384,7 +384,7 @@ class DataQualityNode(CustomNodeBase): result_df = result_df.with_columns([ pl.col(check["column"]).is_null().alias(f"{check['column']}_has_issues") ]) - + return result_df ``` @@ -394,7 +394,7 @@ class DataQualityNode(CustomNodeBase): class TextCleanerNode(CustomNodeBase): node_name: str = "Text Cleaner" node_category: str = "Text Processing" - + settings_schema: TextCleanerSettings = TextCleanerSettings( cleaning_options=Section( title="Cleaning Options", @@ -425,10 +425,10 @@ class TextCleanerNode(CustomNodeBase): text_col = self.settings_schema.cleaning_options.text_column.value operations = self.settings_schema.cleaning_options.operations.value output_col = self.settings_schema.cleaning_options.output_column.value - + # Start with original text expr = pl.col(text_col) - + # Apply selected operations if "lowercase" in operations: expr = expr.str.to_lowercase() @@ -440,14 +440,14 @@ class TextCleanerNode(CustomNodeBase): expr = expr.str.replace_all(r"\d+", "") if "trim" in operations: expr = expr.str.strip_chars() - + return input_df.with_columns([expr.alias(output_col)]) ``` ## Best Practices ### 1. Performance -Try to use Polars expressions and lazy evaluation to keep your nodes efficient. +Try to use Polars expressions and lazy evaluation to keep your nodes efficient. A collect will be executed in the core process and can cause issues when using remote compute. @@ -484,4 +484,4 @@ The following features are planned for future releases: --- -Ready to build? Start with the [Custom Node Tutorial](custom-node-tutorial.md) for a step-by-step walkthrough! \ No newline at end of file +Ready to build? Start with the [Custom Node Tutorial](custom-node-tutorial.md) for a step-by-step walkthrough! diff --git a/docs/for-developers/custom-node-tutorial.md b/docs/for-developers/custom-node-tutorial.md index a348fad9e..4dc1c5ad3 100644 --- a/docs/for-developers/custom-node-tutorial.md +++ b/docs/for-developers/custom-node-tutorial.md @@ -44,7 +44,7 @@ from typing import List from flowfile_core.flowfile.node_designer import ( CustomNodeBase, - Section, + Section, NodeSettings, TextInput, NumericInput, @@ -70,7 +70,7 @@ class EmojiMoodSection(Section): required=True, data_types=Types.Numeric # Only show numeric columns ) - + mood_type: SingleSelect = SingleSelect( label="Emoji Mood Logic", options=[ @@ -84,14 +84,14 @@ class EmojiMoodSection(Section): ], default="performance" ) - + threshold_value: NumericInput = NumericInput( label="Mood Threshold", default=50.0, min_value=0, max_value=100 ) - + emoji_column_name: TextInput = TextInput( label="New Emoji Column Name", default="mood_emoji", @@ -113,13 +113,13 @@ class EmojiStyleSection(Section): ], default="normal" ) - + add_random_sparkle: ToggleSwitch = ToggleSwitch( label="Add Random Sparkles ✨", default=True, description="Randomly sprinkle ✨ for extra pizzazz" ) - + emoji_categories: MultiSelect = MultiSelect( label="Allowed Emoji Categories", options=[ @@ -143,7 +143,7 @@ class EmojiSettings(NodeSettings): title="Mood Detection 😊", description="Configure how to detect the vibe of your data" ) - + style_options: EmojiStyleSection = EmojiStyleSection( title="Emoji Style 🎨", description="Fine-tune your emoji experience" @@ -160,11 +160,11 @@ class EmojiGenerator(CustomNodeBase): node_group: str = "custom" title: str = "Emoji Generator" intro: str = "Transform boring data into fun emoji-filled datasets! 🚀" - + # I/O configuration number_of_inputs: int = 1 number_of_outputs: int = 1 - + # Link to our settings schema settings_schema: EmojiSettings = EmojiSettings() @@ -573,10 +573,9 @@ Save this as `~/.flowfile/user_defined_nodes/emoji_generator.py`, restart Flowfi ## Congratulations! 🎉 -You've successfully created a fully functional custom node +You've successfully created a fully functional custom node - ✅ Multi-section UI with 6 different component types - ✅ Complex processing logic with multiple mood themes - ✅ Advanced features like intensity control and random effects - ✅ Professional documentation and structure - diff --git a/docs/for-developers/design-philosophy.md b/docs/for-developers/design-philosophy.md index e79bc84e7..59188fba2 100644 --- a/docs/for-developers/design-philosophy.md +++ b/docs/for-developers/design-philosophy.md @@ -84,7 +84,7 @@ graph: FlowGraph = ff.create_flow_graph() df_1 = ff.FlowFrame(raw_data, flow_graph=graph) df_2 = df_1.with_columns( - flowfile_formulas=['[quantity] * [price]'], + flowfile_formulas=['[quantity] * [price]'], output_column_names=["total"] ) @@ -103,9 +103,9 @@ df_4 = df_3.group_by(['region']).agg([ ```python # Access all nodes that were created in the graph print(graph._node_db) -# {1: Node id: 1 (manual_input), -# 3: Node id: 3 (formula), -# 4: Node id: 4 (filter), +# {1: Node id: 1 (manual_input), +# 3: Node id: 3 (formula), +# 4: Node id: 4 (filter), # 5: Node id: 5 (group_by)} # Find the starting node(s) of the graph @@ -118,7 +118,7 @@ print(graph.get_node(1).leads_to_nodes) # The other way around works too print(graph.get_node(3).node_inputs) -# NodeStepInputs(Left Input: None, Right Input: None, +# NodeStepInputs(Left Input: None, Right Input: None, # Main Inputs: [Node id: 1 (manual_input)]) # Access the settings and type of any node @@ -138,7 +138,7 @@ flow = ff.create_flow_graph() # Node 1: Manual input node_manual_input = node_interface.NodeManualInput( - flow_id=flow.flow_id, + flow_id=flow.flow_id, node_id=1, raw_data_format=RawData.from_pylist(raw_data) ) @@ -150,14 +150,14 @@ formula_node = node_interface.NodeFormula( node_id=2, function=transformation_settings.FunctionInput( field=transformation_settings.FieldInput( - name="total", + name="total", data_type="Double" ), function="[quantity] * [price]" ) ) flow.add_formula(formula_node) -add_connection(flow, +add_connection(flow, node_interface.NodeConnection.create_from_simple_input(1, 2)) # Node 3: Filter high value transactions @@ -170,7 +170,7 @@ filter_node = node_interface.NodeFilter( ) ) flow.add_filter(filter_node) -add_connection(flow, +add_connection(flow, node_interface.NodeConnection.create_from_simple_input(2, 3)) # Node 4: Group by region @@ -186,7 +186,7 @@ group_by_node = node_interface.NodeGroupBy( ) ) flow.add_group_by(group_by_node) -add_connection(flow, +add_connection(flow, node_interface.NodeConnection.create_from_simple_input(3, 4)) ``` @@ -194,8 +194,8 @@ add_connection(flow, ```python # Check the schema at any node print([s.get_minimal_field_info() for s in flow.get_node(4).schema]) -# [MinimalFieldInfo(name='region', data_type='String'), -# MinimalFieldInfo(name='total_revenue', data_type='Float64'), +# [MinimalFieldInfo(name='region', data_type='String'), +# MinimalFieldInfo(name='total_revenue', data_type='Float64'), # MinimalFieldInfo(name='avg_transaction', data_type='Float64')] ``` @@ -206,7 +206,7 @@ This is the polars query plan generated by both methods: ``` AGGREGATE[maintain_order: false] - [col("total").sum().alias("total_revenue"), + [col("total").sum().alias("total_revenue"), col("total").mean().alias("avg_transaction")] BY [col("region")] FROM FILTER [(col("total")) > (1500)] @@ -270,7 +270,7 @@ class AggColl: !!! tip "Settings Power The Backend" This dual structure—Nodes for graph metadata, Settings for transformation logic—drives the backend: - + - 🔧 **Code generation** (method signatures match settings) - 💾 **Serialization** (graphs can be saved/loaded) - 🔮 **Schema prediction** (output types are inferred from AggColl) @@ -295,21 +295,21 @@ The `FlowNode` class is the heart of each transformation in the graph. Each node !!! info "Core FlowNode Components" **Essential State:** - + - **`_function`**: The closure containing the transformation logic - **`leads_to_nodes`**: List of downstream nodes that depend on this one - **`node_information`**: Metadata (id, type, position, connections) - **`_hash`**: Unique identifier based on settings and parent hashes - + **Runtime State:** - + - **`results`**: Holds the resulting data, errors, and example data paths - **`node_stats`**: Tracks execution status (has_run, is_canceled, etc.) - **`node_settings`**: Runtime settings (cache_results, streamable, etc.) - **`state_needs_reset`**: Flag indicating if the node needs recalculation - + **Schema Information:** - + - **`node_schema`**: Input/output columns and predicted schemas - **`schema_callback`**: Function to calculate schema without execution @@ -370,7 +370,7 @@ def add_group_by(self, group_by_settings: input_schema.NodeGroupBy): # The closure: captures group_by_settings def _func(fl: FlowDataEngine) -> FlowDataEngine: return fl.do_group_by(group_by_settings.groupby_input, False) - + self.add_node_step( node_id=group_by_settings.node_id, function=_func, # This closure remembers group_by_settings! @@ -384,7 +384,7 @@ def add_union(self, union_settings: input_schema.NodeUnion): def _func(*flowfile_tables: FlowDataEngine): dfs = [flt.data_frame for flt in flowfile_tables] return FlowDataEngine(pl.concat(dfs, how='diagonal_relaxed')) - + self.add_node_step( node_id=union_settings.node_id, function=_func, # This closure has everything it needs @@ -416,7 +416,7 @@ print(result.data_frame.collect_schema()) 2. **Functions only need FlowDataEngine as input** (or multiple for joins/unions) 3. **LazyFrame tracks schema changes** through the entire chain 4. **No data is processed**—Polars just builds the query plan - + The result: instant schema feedback without running expensive computations! ### Fallback: Schema Callbacks @@ -506,16 +506,16 @@ graph LR subgraph "Frontend" A[Designer
Vue/Electron] end - + subgraph "Backend" B[Core Service
FastAPI] C[Worker Service
FastAPI] end - + subgraph "Storage" D[Arrow IPC
Cache] end - + A <-->|Settings/Schema| B B <-->|Execution| C C <-->|Data| D @@ -526,12 +526,12 @@ graph LR - Visual graph building interface - Node configuration forms (manually implemented) - Real-time schema feedback - + **Core:** - DAG management - Execution orchestration - Schema prediction - + **Worker:** - Polars transformations - Data caching (Arrow IPC) @@ -557,7 +557,7 @@ flowfile/ !!! warning "Current State of Node Development" While the backend architecture elegantly uses settings-driven nodes, adding new nodes requires work across multiple layers. The frontend currently requires manual implementation for each node type—the visual editor doesn't automatically generate forms from Pydantic schemas yet. - + However, there are also opportunities for more focused contributions! Integration with databases and cloud services is needed—these are smaller, more targeted tasks since the core structure is already in place. There's a lot of active development happening, so it's an exciting time to contribute! ### Adding a New Node: The Full Picture @@ -580,7 +580,7 @@ def add_custom_transform(self, transform_settings: input_schema.NodeCustomTransf # Create the closure that captures settings def _func(fl: FlowDataEngine) -> FlowDataEngine: return fl.do_custom_transform(transform_settings.transform_input) - + # Register with the graph self.add_node_step( node_id=transform_settings.node_id, @@ -589,7 +589,7 @@ def add_custom_transform(self, transform_settings: input_schema.NodeCustomTransf setting_input=transform_settings, input_node_ids=[transform_settings.depending_on_id] ) - + # Don't forget schema prediction! node = self.get_node(transform_settings.node_id) # ... schema callback setup ... @@ -610,7 +610,7 @@ This manual process ensures full control over the UI/UX but requires significant The goal is to eventually auto-generate UI from Pydantic schemas, which would complete the settings-driven architecture. This would make adding new nodes closer to just defining the backend settings and transformation logic, with the UI automatically following. -The beauty of Flowfile's architecture—discovered through the organic evolution from a UI-first approach—is that even though adding nodes requires work across multiple layers today, the settings-based design provides a clear contract between visual and code interfaces. +The beauty of Flowfile's architecture—discovered through the organic evolution from a UI-first approach—is that even though adding nodes requires work across multiple layers today, the settings-based design provides a clear contract between visual and code interfaces. I hope you enjoyed learning about Flowfile's architecture and found the dual-interface approach as exciting as I do! If you have questions, ideas, or want to contribute, ] -feel free to reach out via [GitHub](https://github.com/edwardvaneechoud/Flowfile) or check our [Core Developer Guide](flowfile-core.md). Happy building! \ No newline at end of file +feel free to reach out via [GitHub](https://github.com/edwardvaneechoud/Flowfile) or check our [Core Developer Guide](flowfile-core.md). Happy building! diff --git a/docs/for-developers/flowfile-core.md b/docs/for-developers/flowfile-core.md index c14edef5f..f9dad6f55 100644 --- a/docs/for-developers/flowfile-core.md +++ b/docs/for-developers/flowfile-core.md @@ -72,7 +72,7 @@ Settings: #<-- The FlowSettings object you provid ``` -```python +```python print(graph.run_graph()) # flow_id=1 start_time=datetime.datetime(...) end_time=datetime.datetime(...) success=True nodes_completed=0 number_of_nodes=0 node_step_result=[] diff --git a/docs/for-developers/index.md b/docs/for-developers/index.md index b4d92535a..c2e1189f3 100644 --- a/docs/for-developers/index.md +++ b/docs/for-developers/index.md @@ -80,4 +80,3 @@ We welcome contributions! Adding a new node requires changes across the stack: - **Frontend**: Currently, you must also manually create a Vue component for the node's configuration form in the visual editor. For a more detailed breakdown, please read the **[Contributing section in our Design Philosophy guide](design-philosophy.md#contributing)**. - diff --git a/docs/for-developers/python-api-reference.md b/docs/for-developers/python-api-reference.md index c171bf120..ce1161d3d 100644 --- a/docs/for-developers/python-api-reference.md +++ b/docs/for-developers/python-api-reference.md @@ -17,7 +17,7 @@ The `FlowGraph` is the central object that orchestrates the execution of data tr show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -32,7 +32,7 @@ The `FlowNode` represents a single operation in the `FlowGraph`. Each node corre show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -47,7 +47,7 @@ The `FlowDataEngine` is the primary engine of the library, providing a rich API show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -83,7 +83,7 @@ This section documents the Pydantic models that define the structure of settings show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -97,7 +97,7 @@ This section documents the Pydantic models that define the structure of settings show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -111,7 +111,7 @@ This section documents the Pydantic models that define the structure of settings show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -125,7 +125,7 @@ This section documents the Pydantic models that define the structure of settings show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -139,7 +139,7 @@ This section documents the Pydantic models that define the structure of settings show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -158,7 +158,7 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -171,7 +171,7 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -185,7 +185,7 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -198,7 +198,7 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -211,7 +211,7 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true @@ -224,9 +224,9 @@ This section documents the FastAPI routes that expose `flowfile-core`'s function show_signature: true show_source: true heading_level: 4 - show_symbol_type_heading: true + show_symbol_type_heading: true show_root_members_full_path: false summary: true unwrap_annotated: true show_symbol_type_toc: true ---- \ No newline at end of file +--- diff --git a/docs/index.html b/docs/index.html index a957f70b4..6855dc381 100644 --- a/docs/index.html +++ b/docs/index.html @@ -957,4 +957,4 @@

See It For Yourself

} - \ No newline at end of file + diff --git a/docs/quickstart.md b/docs/quickstart.md index 4edaf6954..f447db14b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -20,7 +20,7 @@ Download the latest installer for your platform: - **Windows**: [Flowfile-Setup.exe](https://github.com/Edwardvaneechoud/Flowfile/releases) -- **macOS**: [Flowfile.dmg](https://github.com/Edwardvaneechoud/Flowfile/releases) +- **macOS**: [Flowfile.dmg](https://github.com/Edwardvaneechoud/Flowfile/releases) > **Note**: You may see security warnings since the installer isn't signed. On Windows, click "More info" → "Run anyway". On macOS, right-click → "Open" → confirm. @@ -59,14 +59,14 @@ npm run dev:web # Terminal 3 (port 8080)

Non-Technical Users

Perfect for: Analysts, business users, Excel power users

No coding required!

- + - + Start Visual Tutorial → @@ -74,14 +74,14 @@ npm run dev:web # Terminal 3 (port 8080)

Technical Users

Perfect for: Developers, data scientists, engineers

Full programmatic control!

- + - + Start Python Tutorial → @@ -292,28 +292,28 @@ import flowfile as ff pipeline = ( # Extract: Read partitioned parquet files from S3 ff.scan_parquet_from_cloud_storage( - "s3://data-lake/sales/year=2024/month=*", + "s3://data-lake/sales/year=2024/month=*", connection_name="production-data", description="Load Q1-Q4 2024 sales data" ) - + # Transform: Clean and enrich .filter( - (ff.col("status") == "completed") & + (ff.col("status") == "completed") & (ff.col("amount") > 0), description="Keep only valid completed transactions" ) - + # Add calculated fields .with_columns([ # Business logic (ff.col("amount") * ff.col("quantity")).alias("line_total"), (ff.col("amount") * ff.col("quantity") * 0.1).alias("tax"), - + # Date features for analytics ff.col("order_date").dt.quarter().alias("quarter"), ff.col("order_date").dt.day_of_week().alias("day_of_week"), - + # Customer segmentation ff.when(ff.col("customer_lifetime_value") > 10000) .then(ff.lit("VIP")) @@ -321,7 +321,7 @@ pipeline = ( .then(ff.lit("Regular")) .otherwise(ff.lit("New")) .alias("customer_segment"), - + # Region mapping ff.when(ff.col("state").is_in(["CA", "OR", "WA"])) .then(ff.lit("West")) @@ -332,27 +332,27 @@ pipeline = ( .otherwise(ff.lit("Midwest")) .alias("region") ], description="Add business metrics and segments") - + # Complex aggregation .group_by(["region", "quarter", "customer_segment"]) .agg([ # Revenue metrics ff.col("line_total").sum().alias("total_revenue"), ff.col("tax").sum().alias("total_tax"), - + # Order metrics ff.col("order_id").n_unique().alias("unique_orders"), ff.col("customer_id").n_unique().alias("unique_customers"), - + # Performance metrics ff.col("line_total").mean().round(2).alias("avg_order_value"), ff.col("quantity").sum().alias("units_sold"), - + # Statistical metrics ff.col("line_total").std().round(2).alias("revenue_std"), ff.col("line_total").quantile(0.5).alias("median_order_value") ]) - + # Final cleanup .sort(["region", "quarter", "total_revenue"], descending=[False, False, True]) .filter(ff.col("total_revenue") > 1000) # Remove noise @@ -435,27 +435,27 @@ import flowfile as ff def data_quality_pipeline(df: ff.FlowFrame) -> ff.FlowFrame: """Reusable data quality check pipeline""" - + # Record initial count initial_count = df.select(ff.col("*").count().alias("count")) - + # Apply quality filters clean_df = ( df # Remove nulls in critical fields .drop_nulls(subset=["order_id", "customer_id", "amount"]) - + # Validate data ranges .filter( (ff.col("amount").is_between(0, 1000000)) & (ff.col("quantity") > 0) & (ff.col("order_date") <= datetime.now()) ) - + # Remove duplicates .unique(subset=["order_id"], keep="first") ) - + # Log quality metrics final_count = clean_df.select(ff.col("*").count().alias("count")) print(f"Initial count: {initial_count.collect()[0]['count']}") @@ -529,9 +529,9 @@ enriched_orders = ( # Handle missing values from left joins ff.col("customer_segment").fill_null("Unknown"), ff.col("product_category").fill_null("Other"), - + # Calculate metrics - (ff.col("unit_price") * ff.col("quantity") * + (ff.col("unit_price") * ff.col("quantity") * (ff.lit(1) - ff.col("discount_rate").fill_null(0))).alias("net_revenue") ]) ) @@ -638,4 +638,4 @@ FLOWFILE_PORT=8080 flowfile run ui Start Visual (No Code) → Start Coding (Python) → - \ No newline at end of file + diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 379b69007..2d7b15a58 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -1,4 +1,4 @@ /* docs/stylesheets/extra.css */ .md-grid { max-width: 1920px; -} \ No newline at end of file +} diff --git a/docs/users/index.md b/docs/users/index.md index 7cdc43ae0..30a9d2abc 100644 --- a/docs/users/index.md +++ b/docs/users/index.md @@ -77,4 +77,4 @@ ff.open_graph_in_editor(result.flow_graph) # See it visually! --- -*Remember: Every visual flow can become code, and every code pipeline can be visualized. Choose what feels natural and switch whenever you want!* \ No newline at end of file +*Remember: Every visual flow can become code, and every code pipeline can be visualized. Choose what feels natural and switch whenever you want!* diff --git a/docs/users/python-api/concepts/design-concepts.md b/docs/users/python-api/concepts/design-concepts.md index 85ea18fbb..8c514197e 100644 --- a/docs/users/python-api/concepts/design-concepts.md +++ b/docs/users/python-api/concepts/design-concepts.md @@ -36,7 +36,7 @@ A `FlowFrame` never loads your actual data into memory until you explicitly call df = ( ff.FlowFrame({ "id": [1, 2, 3, 4, 5], - "amount": [500, 1200, 800, 1500, 900], + "amount": [500, 1200, 800, 1500, 900], "category": ["A", "B", "A", "C", "B"] }) # Creates manual input node .filter(ff.col("amount") > 1000) # No filtering happens yet @@ -49,7 +49,7 @@ result = df.collect() # Everything executes at once, optimized ``` !!! info "Performance Benefits" - This lazy evaluation is powered by Polars and explained in detail in our [Technical Architecture guide](../../../for-developers/architecture.md#the-power-of-lazy-evaluation). + This lazy evaluation is powered by Polars and explained in detail in our [Technical Architecture guide](../../../for-developers/architecture.md#the-power-of-lazy-evaluation). #### 2. Connected to a DAG (Directed Acyclic Graph) Every FlowFrame has a reference to a FlowGraph that tracks every operation as a node: @@ -264,4 +264,4 @@ Understanding this design helps you build efficient, maintainable data pipelines - **[Joins](../reference/joins.md)** - Combining datasets - **[Aggregations](../reference/aggregations.md)** - Group by and summarization - **[Visual UI Integration](../reference/visual-ui.md)** - Working with the visual editor -- **[Developers guide](../../../for-developers/index.md)** - Core architecture and design philosophy \ No newline at end of file +- **[Developers guide](../../../for-developers/index.md)** - Core architecture and design philosophy diff --git a/docs/users/python-api/concepts/index.md b/docs/users/python-api/concepts/index.md index a720b1f50..1a7f863da 100644 --- a/docs/users/python-api/concepts/index.md +++ b/docs/users/python-api/concepts/index.md @@ -95,4 +95,4 @@ Understanding these concepts helps you: --- -*These concepts are the foundation of Flowfile. Understanding them will make everything else click!* \ No newline at end of file +*These concepts are the foundation of Flowfile. Understanding them will make everything else click!* diff --git a/docs/users/python-api/index.md b/docs/users/python-api/index.md index aa3ec5f19..9297376e1 100644 --- a/docs/users/python-api/index.md +++ b/docs/users/python-api/index.md @@ -3,7 +3,7 @@ Build data pipelines programmatically with Flowfile's Polars-compatible API. -!!! info "If You Know Polars, You Know Flowfile" +!!! info "If You Know Polars, You Know Flowfile" Our API is designed to be a seamless extension of Polars. The majority of the methods are identical, so you can leverage your existing knowledge to be productive from day one. The main additions are features that connect your code to the broader Flowfile ecosystem, like cloud integrations and UI visualization. @@ -59,4 +59,4 @@ Want to understand how Flowfile works internally or contribute to the project? S --- -*Prefer visual workflows? Check out the [Visual Editor Guide](../visual-editor/index.md).* \ No newline at end of file +*Prefer visual workflows? Check out the [Visual Editor Guide](../visual-editor/index.md).* diff --git a/docs/users/python-api/reference/cloud-connections.md b/docs/users/python-api/reference/cloud-connections.md index 95d95f413..c9438e137 100644 --- a/docs/users/python-api/reference/cloud-connections.md +++ b/docs/users/python-api/reference/cloud-connections.md @@ -1,7 +1,7 @@ # Cloud Connection Management -Flowfile provides secure, centralized management for cloud storage connections. Connections can be created through code or the UI—both store credentials in an encrypted database. -On this page we will cover how to create and manage them in Python. If you want to learn how to create them in the UI, +Flowfile provides secure, centralized management for cloud storage connections. Connections can be created through code or the UI—both store credentials in an encrypted database. +On this page we will cover how to create and manage them in Python. If you want to learn how to create them in the UI, check out the [UI guide](../../visual-editor/tutorials/cloud-connections.md). ## Creating Connections diff --git a/docs/users/python-api/reference/index.md b/docs/users/python-api/reference/index.md index 6591177e4..63a5e396b 100644 --- a/docs/users/python-api/reference/index.md +++ b/docs/users/python-api/reference/index.md @@ -69,4 +69,4 @@ For understanding how Flowfile works internally: --- -*This reference covers Flowfile-specific features. For standard Polars operations, see the [Polars API Reference](https://pola-rs.github.io/polars/py-polars/html/reference/).* \ No newline at end of file +*This reference covers Flowfile-specific features. For standard Polars operations, see the [Polars API Reference](https://pola-rs.github.io/polars/py-polars/html/reference/).* diff --git a/docs/users/python-api/reference/visual-ui.md b/docs/users/python-api/reference/visual-ui.md index b65667673..1d7768845 100644 --- a/docs/users/python-api/reference/visual-ui.md +++ b/docs/users/python-api/reference/visual-ui.md @@ -55,7 +55,7 @@ ff.open_graph_in_editor(result.flow_graph) When you call `open_graph_in_editor()`: -1. **Saves the graph** to a temporary `.flowfile` +1. **Saves the graph** to a temporary `.flowfile` 2. **Checks if server is running** at `http://localhost:63578` 3. **Starts server if needed** using `flowfile run ui --no-browser` 4. **Imports the flow** via API endpoint @@ -91,7 +91,7 @@ ff.open_graph_in_editor( # All server management functions are in flowfile.api from flowfile.api import ( is_flowfile_running, - start_flowfile_server_process, + start_flowfile_server_process, stop_flowfile_server_process, get_auth_token ) @@ -232,4 +232,4 @@ ff.open_graph_in_editor(pipeline3.flow_graph) # Still same server - **Explore Visual Nodes:** Learn the details of each node available in the [Visual Editor](../../visual-editor/nodes/index.md). - **Convert Code to Visual:** See how your code translates into a visual workflow in the [Conversion Guide](../tutorials/flowfile_frame_api.md). - **Build with Code:** Dive deeper into the [code-first approach](../../visual-editor/building-flows.md) for building pipelines. - - **Back to Index:** Return to the main [Python API Index](index.md). \ No newline at end of file + - **Back to Index:** Return to the main [Python API Index](index.md). diff --git a/docs/users/python-api/reference/writing-data.md b/docs/users/python-api/reference/writing-data.md index 9f8f4a45d..1edb1234b 100644 --- a/docs/users/python-api/reference/writing-data.md +++ b/docs/users/python-api/reference/writing-data.md @@ -94,7 +94,7 @@ df.write_parquet_to_cloud_storage( ```python # Write JSON to cloud storage df.write_json_to_cloud_storage( - "s3://api-data/export.json", + "s3://api-data/export.json", connection_name="api-storage", description="Export for API consumption" ) @@ -114,7 +114,7 @@ df.write_delta( # Append to existing Delta table new_data.write_delta( "s3://warehouse/customer_dim", - connection_name="warehouse-connection", + connection_name="warehouse-connection", write_mode="append", description="Add new customers to dimension" ) @@ -161,7 +161,7 @@ from pydantic import SecretStr ff.create_cloud_storage_connection_if_not_exists( ff.FullCloudStorageConnection( connection_name="data-lake", - storage_type="s3", + storage_type="s3", auth_method="access_key", aws_region="us-east-1", aws_access_key_id="your-key", @@ -177,6 +177,6 @@ df.write_parquet_to_cloud_storage( ``` ---- +--- [← Previous: Reading Data](reading-data.md) | [Next: Data Types →](data-types.md) diff --git a/docs/users/python-api/tutorials/flowfile_frame_api.md b/docs/users/python-api/tutorials/flowfile_frame_api.md index b9ea8f45b..41d0b55a8 100644 --- a/docs/users/python-api/tutorials/flowfile_frame_api.md +++ b/docs/users/python-api/tutorials/flowfile_frame_api.md @@ -104,4 +104,4 @@ By combining the declarative power of a Polars-like API with Flowfile’s intera * Code-first development with automatic visualization * Zero-config ETL graph generation -* Easy debugging and collaboration \ No newline at end of file +* Easy debugging and collaboration diff --git a/docs/users/python-api/tutorials/index.md b/docs/users/python-api/tutorials/index.md index 4269ca1e8..a42870c2c 100644 --- a/docs/users/python-api/tutorials/index.md +++ b/docs/users/python-api/tutorials/index.md @@ -86,4 +86,4 @@ null_counts = df.select([ --- -*Want more tutorials? Let us know what you'd like to see in our [GitHub Discussions](https://github.com/edwardvaneechoud/Flowfile/discussions)!* \ No newline at end of file +*Want more tutorials? Let us know what you'd like to see in our [GitHub Discussions](https://github.com/edwardvaneechoud/Flowfile/discussions)!* diff --git a/docs/users/visual-editor/building-flows.md b/docs/users/visual-editor/building-flows.md index 7fc566d8f..b82e7c63e 100644 --- a/docs/users/visual-editor/building-flows.md +++ b/docs/users/visual-editor/building-flows.md @@ -11,7 +11,7 @@ Flowfile allows you to create data pipelines visually by connecting nodes that r *The complete Flowfile interface showing:* -- **Left sidebar**: Browse and select from available nodes +- **Left sidebar**: Browse and select from available nodes - **Center canvas**: Build your flow by arranging and connecting nodes - **Right sidebar**: Configure node settings and parameters - **Bottom panel**: Preview data at each step @@ -172,7 +172,7 @@ Here's a typical flow that demonstrates common operations: --- --- -## Want to see another example? +## Want to see another example? Checkout the [quickstart guide](../../quickstart.md#quick-start-for-non-technical-users-non-technical-quickstart)! ## Next Steps @@ -183,4 +183,4 @@ After mastering basic flows, explore: - [Complex transformations](nodes/transform.md) - [Data aggregation techniques](nodes/aggregate.md) - [Advanced joining methods](nodes/combine.md) - - [Output options](nodes/output.md) \ No newline at end of file + - [Output options](nodes/output.md) diff --git a/docs/users/visual-editor/index.md b/docs/users/visual-editor/index.md index 793c10128..cb83ec145 100644 --- a/docs/users/visual-editor/index.md +++ b/docs/users/visual-editor/index.md @@ -92,4 +92,4 @@ Remember, you can always switch between them! --- -*Ready to build? Start with [Building Flows](building-flows.md) or explore the [Node Reference](nodes/index.md).* \ No newline at end of file +*Ready to build? Start with [Building Flows](building-flows.md) or explore the [Node Reference](nodes/index.md).* diff --git a/docs/users/visual-editor/nodes/aggregate.md b/docs/users/visual-editor/nodes/aggregate.md index 79cb913bc..c710fd730 100644 --- a/docs/users/visual-editor/nodes/aggregate.md +++ b/docs/users/visual-editor/nodes/aggregate.md @@ -123,4 +123,4 @@ This node has **no additional settings**—it simply returns the record count. This transformation is useful for **quick dataset validation** and **workflow monitoring**. --- -[← Combine data](combine.md) | [Next: Write data →](output.md) \ No newline at end of file +[← Combine data](combine.md) | [Next: Write data →](output.md) diff --git a/docs/users/visual-editor/nodes/combine.md b/docs/users/visual-editor/nodes/combine.md index 601ccef99..5fce2003d 100644 --- a/docs/users/visual-editor/nodes/combine.md +++ b/docs/users/visual-editor/nodes/combine.md @@ -152,4 +152,4 @@ The **Graph Solver** node groups related records based on connections in a graph This node is useful for **detecting dependencies, clustering related entities, and analyzing network connections**. --- -[← Transform data](transform.md) | [Next: Aggregate data →](aggregate.md) \ No newline at end of file +[← Transform data](transform.md) | [Next: Aggregate data →](aggregate.md) diff --git a/docs/users/visual-editor/nodes/output.md b/docs/users/visual-editor/nodes/output.md index 0cee365a2..ff0365c02 100644 --- a/docs/users/visual-editor/nodes/output.md +++ b/docs/users/visual-editor/nodes/output.md @@ -117,4 +117,3 @@ The **Cloud Storage Writer** node allows you to save your processed data directl ### ![Explore Data](../../../assets/images/nodes/explore_data.png){ width="50" height="50" } Explore Data The Explore Data node provides interactive data exploration and analysis capabilities. - diff --git a/docs/users/visual-editor/nodes/transform.md b/docs/users/visual-editor/nodes/transform.md index 30ebfd7bd..3fd2445d2 100644 --- a/docs/users/visual-editor/nodes/transform.md +++ b/docs/users/visual-editor/nodes/transform.md @@ -241,7 +241,7 @@ The **Text to Rows** node splits text from a selected column into multiple rows #### **Usage** 1. Select the **column to split**. 2. Choose a **delimiter** or use another column for splitting. -3. (Optional) Set an **output column name**. +3. (Optional) Set an **output column name**. --- diff --git a/docs/users/visual-editor/tutorials/code-generator.md b/docs/users/visual-editor/tutorials/code-generator.md index 27c8fef1b..cf503f3a6 100644 --- a/docs/users/visual-editor/tutorials/code-generator.md +++ b/docs/users/visual-editor/tutorials/code-generator.md @@ -127,4 +127,4 @@ if __name__ == "__main__": -These examples provide a clear overview of the type of high-quality, executable Python code produced by Flowfile's Code Generator. \ No newline at end of file +These examples provide a clear overview of the type of high-quality, executable Python code produced by Flowfile's Code Generator. diff --git a/docs/users/visual-editor/tutorials/index.md b/docs/users/visual-editor/tutorials/index.md index eae7257bc..4670e7ea3 100644 --- a/docs/users/visual-editor/tutorials/index.md +++ b/docs/users/visual-editor/tutorials/index.md @@ -60,4 +60,4 @@ Each tutorial includes: --- -*Need help? Check the [Node Reference](../nodes/index.md) for detailed documentation on each node type.* \ No newline at end of file +*Need help? Check the [Node Reference](../nodes/index.md) for detailed documentation on each node type.* diff --git a/flowfile/readme.md b/flowfile/readme.md index b5b9caead..07a06a57d 100644 --- a/flowfile/readme.md +++ b/flowfile/readme.md @@ -124,4 +124,4 @@ Once you're familiar with the web UI, you might want to explore: 1. The desktop application for a more native experience 2. Docker deployment for production environments 3. Advanced ETL operations using the FlowFrame API -4. Custom node development for specialized transformations \ No newline at end of file +4. Custom node development for specialized transformations diff --git a/flowfile_core/Dockerfile b/flowfile_core/Dockerfile index d38505392..3559c7d77 100644 --- a/flowfile_core/Dockerfile +++ b/flowfile_core/Dockerfile @@ -45,4 +45,4 @@ ENV FLOWFILE_MODE=docker WORKDIR /app # Command to run the application -CMD ["python", "-m", "flowfile_core.main"] \ No newline at end of file +CMD ["python", "-m", "flowfile_core.main"] diff --git a/flowfile_core/README.md b/flowfile_core/README.md index dd4b9abad..dda185874 100644 --- a/flowfile_core/README.md +++ b/flowfile_core/README.md @@ -83,4 +83,4 @@ Contributions are welcome! Please read our contributing guidelines before submit ## 📝 License -[MIT License](LICENSE) \ No newline at end of file +[MIT License](LICENSE) diff --git a/flowfile_core/flowfile_core/__init__.py b/flowfile_core/flowfile_core/__init__.py index fa6067d5b..b7e178626 100644 --- a/flowfile_core/flowfile_core/__init__.py +++ b/flowfile_core/flowfile_core/__init__.py @@ -14,6 +14,7 @@ init_db() + class ServerRun: exit: bool = False diff --git a/flowfile_core/flowfile_core/auth/jwt.py b/flowfile_core/flowfile_core/auth/jwt.py index c10ae359e..381a9023f 100644 --- a/flowfile_core/flowfile_core/auth/jwt.py +++ b/flowfile_core/flowfile_core/auth/jwt.py @@ -81,7 +81,7 @@ def get_current_user_sync(token: str, db: Session): full_name=user.full_name, disabled=user.disabled, is_admin=user.is_admin, - must_change_password=user.must_change_password + must_change_password=user.must_change_password, ) @@ -129,7 +129,7 @@ async def get_current_user(token: str = Depends(oauth2_scheme), db: Session = De full_name=user.full_name, disabled=user.disabled, is_admin=user.is_admin, - must_change_password=user.must_change_password + must_change_password=user.must_change_password, ) @@ -200,15 +200,12 @@ async def get_current_user_from_query( full_name=user.full_name, disabled=user.disabled, is_admin=user.is_admin, - must_change_password=user.must_change_password + must_change_password=user.must_change_password, ) async def get_current_admin_user(current_user: User = Depends(get_current_user)): """Dependency that requires the current user to be an admin""" if not current_user.is_admin: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Admin privileges required" - ) + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin privileges required") return current_user diff --git a/flowfile_core/flowfile_core/auth/models.py b/flowfile_core/flowfile_core/auth/models.py index d38acfcb4..2afabb48d 100644 --- a/flowfile_core/flowfile_core/auth/models.py +++ b/flowfile_core/flowfile_core/auth/models.py @@ -26,6 +26,7 @@ class UserInDB(User): class UserCreate(BaseModel): """Model for creating a new user (admin only)""" + username: str password: str email: str | None = None @@ -35,6 +36,7 @@ class UserCreate(BaseModel): class UserUpdate(BaseModel): """Model for updating a user (admin only)""" + email: str | None = None full_name: str | None = None disabled: bool | None = None @@ -45,6 +47,7 @@ class UserUpdate(BaseModel): class ChangePassword(BaseModel): """Model for user changing their own password""" + current_password: str new_password: str diff --git a/flowfile_core/flowfile_core/auth/password.py b/flowfile_core/flowfile_core/auth/password.py index f34a44e69..237206019 100644 --- a/flowfile_core/flowfile_core/auth/password.py +++ b/flowfile_core/flowfile_core/auth/password.py @@ -1,6 +1,7 @@ """Password hashing and verification utilities.""" import re + from passlib.context import CryptContext pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") @@ -11,12 +12,13 @@ "min_length": PASSWORD_MIN_LENGTH, "require_number": True, "require_special": True, - "special_chars": "!@#$%^&*()_+-=[]{}|;:,.<>?" + "special_chars": "!@#$%^&*()_+-=[]{}|;:,.<>?", } class PasswordValidationError(Exception): """Raised when password doesn't meet requirements.""" + pass @@ -38,10 +40,10 @@ def validate_password(password: str) -> tuple[bool, str]: if len(password) < PASSWORD_MIN_LENGTH: return False, f"Password must be at least {PASSWORD_MIN_LENGTH} characters long" - if not re.search(r'\d', password): + if not re.search(r"\d", password): return False, "Password must contain at least one number" - if not re.search(r'[!@#$%^&*()_+\-=\[\]{}|;:,.<>?]', password): + if not re.search(r"[!@#$%^&*()_+\-=\[\]{}|;:,.<>?]", password): return False, "Password must contain at least one special character (!@#$%^&*()_+-=[]{}|;:,.<>?)" return True, "" diff --git a/flowfile_core/flowfile_core/configs/flow_logger.py b/flowfile_core/flowfile_core/configs/flow_logger.py index b2b97d0a5..00fcb3370 100644 --- a/flowfile_core/flowfile_core/configs/flow_logger.py +++ b/flowfile_core/flowfile_core/configs/flow_logger.py @@ -133,7 +133,7 @@ def _recreate_impl(self): try: # Create an empty file - with open(self.log_file_path, "w") as f: + with open(self.log_file_path, "w"): pass # Re-setup the logger @@ -224,7 +224,7 @@ def _clear_log_impl(self): # Ensure parent directory exists self.refresh_logger_if_needed() # Truncate file - with open(self.log_file_path, "w") as f: + with open(self.log_file_path, "w"): pass main_logger.info(f"Log file cleared for flow {self.flow_id}") except Exception as e: diff --git a/flowfile_core/flowfile_core/configs/node_store/user_defined_node_registry.py b/flowfile_core/flowfile_core/configs/node_store/user_defined_node_registry.py index a7d1eb4e2..253f0a341 100644 --- a/flowfile_core/flowfile_core/configs/node_store/user_defined_node_registry.py +++ b/flowfile_core/flowfile_core/configs/node_store/user_defined_node_registry.py @@ -153,7 +153,7 @@ def get_custom_nodes_lazy() -> list[type[CustomNodeBase]]: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - for name, obj in inspect.getmembers(module): + for _name, obj in inspect.getmembers(module): if ( inspect.isclass(obj) and issubclass(obj, CustomNodeBase) diff --git a/flowfile_core/flowfile_core/configs/settings.py b/flowfile_core/flowfile_core/configs/settings.py index 4dc9efaef..5444369b3 100644 --- a/flowfile_core/flowfile_core/configs/settings.py +++ b/flowfile_core/flowfile_core/configs/settings.py @@ -94,18 +94,22 @@ def get_default_worker_url(worker_port=None): # Possible values: "electron" (desktop app), "package" (Python package), "docker" (container) FLOWFILE_MODE = os.getenv("FLOWFILE_MODE", "electron") + def is_docker_mode() -> bool: """Check if running in Docker container mode""" return FLOWFILE_MODE == "docker" + def is_electron_mode() -> bool: """Check if running in Electron desktop app mode""" return FLOWFILE_MODE == "electron" + def is_package_mode() -> bool: """Check if running as Python package""" return FLOWFILE_MODE == "package" + # Legacy compatibility - will be removed in future versions IS_RUNNING_IN_DOCKER = is_docker_mode() diff --git a/flowfile_core/flowfile_core/database/init_db.py b/flowfile_core/flowfile_core/database/init_db.py index ae41d0828..c72dbc674 100644 --- a/flowfile_core/flowfile_core/database/init_db.py +++ b/flowfile_core/flowfile_core/database/init_db.py @@ -1,15 +1,17 @@ # Generate a random secure password and hash it +import logging import os import secrets import string -import logging -from sqlalchemy.orm import Session + +from passlib.context import CryptContext from sqlalchemy import text -from flowfile_core.database import models as db_models -from flowfile_core.database.connection import engine, SessionLocal +from sqlalchemy.orm import Session + from flowfile_core.auth.password import get_password_hash +from flowfile_core.database import models as db_models +from flowfile_core.database.connection import SessionLocal, engine -from passlib.context import CryptContext pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") logger = logging.getLogger(__name__) @@ -19,9 +21,7 @@ def run_migrations(): """Run database migrations to update schema for existing databases.""" with engine.connect() as conn: # Check if users table exists - result = conn.execute(text( - "SELECT name FROM sqlite_master WHERE type='table' AND name='users'" - )) + result = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name='users'")) if not result.fetchone(): logger.info("Users table does not exist, will be created with new schema") return @@ -31,14 +31,14 @@ def run_migrations(): columns = [row[1] for row in result.fetchall()] # Add is_admin column if missing - if 'is_admin' not in columns: + if "is_admin" not in columns: logger.info("Adding is_admin column to users table") conn.execute(text("ALTER TABLE users ADD COLUMN is_admin BOOLEAN DEFAULT 0")) conn.commit() logger.info("Migration complete: is_admin column added") # Add must_change_password column if missing - if 'must_change_password' not in columns: + if "must_change_password" not in columns: logger.info("Adding must_change_password column to users table") conn.execute(text("ALTER TABLE users ADD COLUMN must_change_password BOOLEAN DEFAULT 0")) conn.commit() @@ -54,7 +54,7 @@ def run_migrations(): def create_default_local_user(db: Session): local_user = db.query(db_models.User).filter(db_models.User.username == "local_user").first() if not local_user: - random_password = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(32)) + random_password = "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(32)) hashed_password = pwd_context.hash(random_password) local_user = db_models.User( @@ -62,7 +62,7 @@ def create_default_local_user(db: Session): email="local@flowfile.app", full_name="Local User", hashed_password=hashed_password, - must_change_password=False # Local user doesn't need to change password + must_change_password=False, # Local user doesn't need to change password ) db.add(local_user) db.commit() @@ -93,9 +93,7 @@ def create_docker_admin_user(db: Session): return False # Check if user already exists - existing_user = db.query(db_models.User).filter( - db_models.User.username == admin_username - ).first() + existing_user = db.query(db_models.User).filter(db_models.User.username == admin_username).first() if existing_user: # Ensure existing admin user has is_admin=True @@ -115,7 +113,7 @@ def create_docker_admin_user(db: Session): full_name="Admin User", hashed_password=hashed_password, is_admin=True, - must_change_password=True # Force password change on first login + must_change_password=True, # Force password change on first login ) db.add(admin_user) db.commit() @@ -135,4 +133,3 @@ def init_db(): if __name__ == "__main__": init_db() print("Local user created successfully") - diff --git a/flowfile_core/flowfile_core/fileExplorer/funcs.py b/flowfile_core/flowfile_core/fileExplorer/funcs.py index fe58fc546..36fcd2809 100644 --- a/flowfile_core/flowfile_core/fileExplorer/funcs.py +++ b/flowfile_core/flowfile_core/fileExplorer/funcs.py @@ -1,7 +1,7 @@ import os from datetime import datetime from pathlib import Path -from typing import Literal, Optional +from typing import Literal from fastapi import HTTPException from pydantic import BaseModel @@ -135,7 +135,6 @@ def _sanitize_path(self, path: str | Path) -> Path | None: try: # Handle relative paths from current directoryb if isinstance(path, str): - # Remove any suspicious patterns if path.startswith("/"): # For absolute paths or parent references, resolve from sandbox root @@ -369,7 +368,7 @@ def get_files_from_directory( raise type(e)(f"Error scanning directory {dir_name}: {str(e)}") from e -def validate_file_path(user_path: str, allowed_base: Path) -> Optional[Path]: +def validate_file_path(user_path: str, allowed_base: Path) -> Path | None: """Validate a file path is safe and within allowed_base. Uses os.path.realpath + startswith pattern recognized by CodeQL as safe. @@ -383,7 +382,7 @@ def validate_file_path(user_path: str, allowed_base: Path) -> Optional[Path]: """ try: # Block obvious path traversal patterns early - if '..' in user_path: + if ".." in user_path: return None # Get the base path as a normalized, real path string @@ -436,7 +435,7 @@ def validate_path_under_cwd(user_path: str) -> str: if fullpath.startswith(base_path): return fullpath - raise HTTPException(403, 'Access denied') + raise HTTPException(403, "Access denied") # Alias for backward compatibility diff --git a/flowfile_core/flowfile_core/flowfile/code_generator/code_generator.py b/flowfile_core/flowfile_core/flowfile/code_generator/code_generator.py index 1e6903553..71c09a548 100644 --- a/flowfile_core/flowfile_core/flowfile/code_generator/code_generator.py +++ b/flowfile_core/flowfile_core/flowfile/code_generator/code_generator.py @@ -517,7 +517,7 @@ def _handle_left_inner_join_keys( - reverse_action: Mapping to rename __DROP__ columns after join - after_join_drop_cols: Left join keys marked for dropping """ - left_join_keys_to_keep = [jk.new_name for jk in settings.left_select.join_key_selects if jk.keep] + [jk.new_name for jk in settings.left_select.join_key_selects if jk.keep] join_key_duplication_command = [ f'pl.col("{rjk.old_name}").alias("__DROP__{rjk.new_name}__DROP__")' for rjk in settings.right_select.join_key_selects diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py index 28351d53d..8c1ecb036 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py @@ -7,7 +7,7 @@ from copy import deepcopy from dataclasses import dataclass from math import ceil -from typing import Any, Literal, TypeVar, Union +from typing import Any, Literal, TypeVar import polars as pl @@ -171,7 +171,7 @@ class FlowDataEngine: name: str = None number_of_records: int = None errors: list = None - _schema: list["FlowfileColumn"] | None = None + _schema: list[FlowfileColumn] | None = None # Configuration attributes _optimize_memory: bool = False @@ -204,13 +204,17 @@ class FlowDataEngine: def __init__( self, - raw_data: Union[ - list[dict], list[Any], dict[str, Any], "ParquetFile", pl.DataFrame, pl.LazyFrame, input_schema.RawData - ] = None, + raw_data: list[dict] + | list[Any] + | dict[str, Any] + | ParquetFile + | pl.DataFrame + | pl.LazyFrame + | input_schema.RawData = None, path_ref: str = None, name: str = None, optimize_memory: bool = True, - schema: list["FlowfileColumn"] | list[str] | pl.Schema = None, + schema: list[FlowfileColumn] | list[str] | pl.Schema = None, number_of_records: int = None, calculate_schema_stats: bool = False, streamable: bool = True, @@ -273,7 +277,7 @@ def _handle_raw_data(self, raw_data, number_of_records, optimize_memory): self._handle_polars_dataframe(raw_data, number_of_records) elif isinstance(raw_data, pl.LazyFrame): self._handle_polars_lazy_frame(raw_data, number_of_records, optimize_memory) - elif isinstance(raw_data, (list, dict)): + elif isinstance(raw_data, list | dict): self._handle_python_data(raw_data) def _handle_polars_dataframe(self, df: pl.DataFrame, number_of_records: int | None): @@ -303,7 +307,7 @@ def _handle_dict_input(self, data: dict): """(Internal) Initializes the engine from a Python dictionary.""" if len(data) == 0: self.initialize_empty_fl() - lengths = [len(v) if isinstance(v, (list, tuple)) else 1 for v in data.values()] + lengths = [len(v) if isinstance(v, list | tuple) else 1 for v in data.values()] if len(set(lengths)) == 1 and lengths[0] > 1: self.number_of_records = lengths[0] @@ -521,9 +525,7 @@ def _write_json_to_cloud( raise Exception(f"Failed to write JSON to cloud storage: {str(e)}") @classmethod - def from_cloud_storage_obj( - cls, settings: cloud_storage_schemas.CloudStorageReadSettingsInternal - ) -> "FlowDataEngine": + def from_cloud_storage_obj(cls, settings: cloud_storage_schemas.CloudStorageReadSettingsInternal) -> FlowDataEngine: """Creates a FlowDataEngine from an object in cloud storage. This method supports reading from various cloud storage providers like AWS S3, @@ -607,7 +609,7 @@ def _read_iceberg_from_cloud( storage_options: dict[str, Any], credential_provider: Callable | None, read_settings: cloud_storage_schemas.CloudStorageReadSettings, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Reads Iceberg table(s) from cloud storage.""" raise NotImplementedError("Failed to read Iceberg table from cloud storage: Not yet implemented") @@ -618,7 +620,7 @@ def _read_parquet_from_cloud( storage_options: dict[str, Any], credential_provider: Callable | None, is_directory: bool, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Reads Parquet file(s) from cloud storage.""" try: # Use scan_parquet for lazy evaluation @@ -656,7 +658,7 @@ def _read_delta_from_cloud( storage_options: dict[str, Any], credential_provider: Callable | None, read_settings: cloud_storage_schemas.CloudStorageReadSettings, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Reads a Delta Lake table from cloud storage.""" try: logger.info("Reading Delta file from cloud storage...") @@ -687,7 +689,7 @@ def _read_csv_from_cloud( storage_options: dict[str, Any], credential_provider: Callable | None, read_settings: cloud_storage_schemas.CloudStorageReadSettings, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Reads CSV file(s) from cloud storage.""" try: scan_kwargs = { @@ -730,7 +732,7 @@ def _read_json_from_cloud( storage_options: dict[str, Any], credential_provider: Callable | None, is_directory: bool, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Reads JSON file(s) from cloud storage.""" try: if is_directory: @@ -968,7 +970,7 @@ def _create_empty_dataframe(self, n_records: int) -> pl.DataFrame: def do_group_by( self, group_by_input: transform_schemas.GroupByInput, calculate_schema_stats: bool = True - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Performs a group-by operation on the DataFrame. Args: @@ -996,7 +998,7 @@ def do_group_by( calculate_schema_stats=calculate_schema_stats, ) - def do_sort(self, sorts: list[transform_schemas.SortByInput]) -> "FlowDataEngine": + def do_sort(self, sorts: list[transform_schemas.SortByInput]) -> FlowDataEngine: """Sorts the DataFrame by one or more columns. Args: @@ -1015,7 +1017,7 @@ def do_sort(self, sorts: list[transform_schemas.SortByInput]) -> "FlowDataEngine def change_column_types( self, transforms: list[transform_schemas.SelectInput], calculate_schema: bool = False - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Changes the data type of one or more columns. Args: @@ -1110,7 +1112,7 @@ def to_dict(self) -> dict[str, list]: return self.data_frame.to_dict(as_series=False) @classmethod - def create_from_external_source(cls, external_source: ExternalDataSource) -> "FlowDataEngine": + def create_from_external_source(cls, external_source: ExternalDataSource) -> FlowDataEngine: """Creates a FlowDataEngine from an external data source. Args: @@ -1130,7 +1132,7 @@ def create_from_external_source(cls, external_source: ExternalDataSource) -> "Fl return ff @classmethod - def create_from_sql(cls, sql: str, conn: Any) -> "FlowDataEngine": + def create_from_sql(cls, sql: str, conn: Any) -> FlowDataEngine: """Creates a FlowDataEngine by executing a SQL query. Args: @@ -1143,7 +1145,7 @@ def create_from_sql(cls, sql: str, conn: Any) -> "FlowDataEngine": return cls(pl.read_sql(sql, conn)) @classmethod - def create_from_schema(cls, schema: list[FlowfileColumn]) -> "FlowDataEngine": + def create_from_schema(cls, schema: list[FlowfileColumn]) -> FlowDataEngine: """Creates an empty FlowDataEngine from a schema definition. Args: @@ -1160,7 +1162,7 @@ def create_from_schema(cls, schema: list[FlowfileColumn]) -> "FlowDataEngine": return cls(df, schema=schema, calculate_schema_stats=False, number_of_records=0) @classmethod - def create_from_path(cls, received_table: input_schema.ReceivedTable) -> "FlowDataEngine": + def create_from_path(cls, received_table: input_schema.ReceivedTable) -> FlowDataEngine: """Creates a FlowDataEngine from a local file path. Supports various file types like CSV, Parquet, and Excel. @@ -1188,7 +1190,7 @@ def create_from_path(cls, received_table: input_schema.ReceivedTable) -> "FlowDa return flow_file @classmethod - def create_random(cls, number_of_records: int = 1000) -> "FlowDataEngine": + def create_random(cls, number_of_records: int = 1000) -> FlowDataEngine: """Creates a FlowDataEngine with randomly generated data. Useful for testing and examples. @@ -1202,7 +1204,7 @@ def create_random(cls, number_of_records: int = 1000) -> "FlowDataEngine": return cls(create_fake_data(number_of_records)) @classmethod - def generate_enumerator(cls, length: int = 1000, output_name: str = "output_column") -> "FlowDataEngine": + def generate_enumerator(cls, length: int = 1000, output_name: str = "output_column") -> FlowDataEngine: """Generates a FlowDataEngine with a single column containing a sequence of integers. Args: @@ -1265,7 +1267,7 @@ def _handle_string_schema(self, schema: list[str], pl_schema: pl.Schema) -> list return flow_file_columns - def split(self, split_input: transform_schemas.TextToRowsInput) -> "FlowDataEngine": + def split(self, split_input: transform_schemas.TextToRowsInput) -> FlowDataEngine: """Splits a column's text values into multiple rows based on a delimiter. This operation is often referred to as "exploding" the DataFrame, as it @@ -1292,7 +1294,7 @@ def split(self, split_input: transform_schemas.TextToRowsInput) -> "FlowDataEngi return FlowDataEngine(df) - def unpivot(self, unpivot_input: transform_schemas.UnpivotInput) -> "FlowDataEngine": + def unpivot(self, unpivot_input: transform_schemas.UnpivotInput) -> FlowDataEngine: """Converts the DataFrame from a wide to a long format. This is the inverse of a pivot operation, taking columns and transforming @@ -1316,7 +1318,7 @@ def unpivot(self, unpivot_input: transform_schemas.UnpivotInput) -> "FlowDataEng return FlowDataEngine(result) - def do_pivot(self, pivot_input: transform_schemas.PivotInput, node_logger: NodeLogger = None) -> "FlowDataEngine": + def do_pivot(self, pivot_input: transform_schemas.PivotInput, node_logger: NodeLogger = None) -> FlowDataEngine: """Converts the DataFrame from a long to a wide format, aggregating values. Args: @@ -1387,7 +1389,7 @@ def do_pivot(self, pivot_input: transform_schemas.PivotInput, node_logger: NodeL return FlowDataEngine(df, calculate_schema_stats=False) - def do_filter(self, predicate: str) -> "FlowDataEngine": + def do_filter(self, predicate: str) -> FlowDataEngine: """Filters rows based on a predicate expression. Args: @@ -1406,7 +1408,7 @@ def do_filter(self, predicate: str) -> "FlowDataEngine": df = self.data_frame.filter(f) return FlowDataEngine(df, schema=self.schema, streamable=self._streamable) - def add_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine": + def add_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine: """Adds a record ID (row number) column to the DataFrame. Can generate a simple sequential ID or a grouped ID that resets for @@ -1423,7 +1425,7 @@ def add_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> return self._add_grouped_record_id(record_id_settings) return self._add_simple_record_id(record_id_settings) - def _add_grouped_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine": + def _add_grouped_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine: """Adds a record ID column with grouping.""" select_cols = [pl.col(record_id_settings.output_column_name)] + [pl.col(c) for c in self.columns] @@ -1444,7 +1446,7 @@ def _add_grouped_record_id(self, record_id_settings: transform_schemas.RecordIdI return FlowDataEngine(df, schema=output_schema) - def _add_simple_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine": + def _add_simple_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine: """Adds a simple sequential record ID column.""" df = self.data_frame.with_row_index(record_id_settings.output_column_name, record_id_settings.offset) @@ -1482,7 +1484,7 @@ def __repr__(self) -> str: """Returns a string representation of the FlowDataEngine.""" return f"flow data engine\n{self.data_frame.__repr__()}" - def __call__(self) -> "FlowDataEngine": + def __call__(self) -> FlowDataEngine: """Makes the class instance callable, returning itself.""" return self @@ -1490,7 +1492,7 @@ def __len__(self) -> int: """Returns the number of records in the table.""" return self.number_of_records if self.number_of_records >= 0 else self.get_number_of_records() - def cache(self) -> "FlowDataEngine": + def cache(self) -> FlowDataEngine: """Caches the current DataFrame to disk and updates the internal reference. This triggers a background process to write the current LazyFrame's result @@ -1545,7 +1547,7 @@ def get_output_sample(self, n_rows: int = 10) -> list[dict]: df = self.collect() return df.to_dicts() - def __get_sample__(self, n_rows: int = 100, streamable: bool = True) -> "FlowDataEngine": + def __get_sample__(self, n_rows: int = 100, streamable: bool = True) -> FlowDataEngine: """Internal method to get a sample of the data.""" if not self.lazy: df = self.data_frame.lazy() @@ -1569,7 +1571,7 @@ def get_sample( shuffle: bool = False, seed: int = None, execution_location: ExecutionLocationsLiteral | None = None, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Gets a sample of rows from the DataFrame. Args: @@ -1608,7 +1610,7 @@ def get_sample( return FlowDataEngine(sample_df, schema=self.schema) - def get_subset(self, n_rows: int = 100) -> "FlowDataEngine": + def get_subset(self, n_rows: int = 100) -> FlowDataEngine: """Gets the first `n_rows` from the DataFrame. Args: @@ -1624,7 +1626,7 @@ def get_subset(self, n_rows: int = 100) -> "FlowDataEngine": def iter_batches( self, batch_size: int = 1000, columns: list | tuple | str = None - ) -> Generator["FlowDataEngine", None, None]: + ) -> Generator[FlowDataEngine, None, None]: """Iterates over the DataFrame in batches. Args: @@ -1645,7 +1647,7 @@ def iter_batches( def start_fuzzy_join( self, fuzzy_match_input: transform_schemas.FuzzyMatchInput, - other: "FlowDataEngine", + other: FlowDataEngine, file_ref: str, flow_id: int = -1, node_id: int | str = -1, @@ -1684,7 +1686,7 @@ def start_fuzzy_join( def fuzzy_join_external( self, fuzzy_match_input: transform_schemas.FuzzyMatchInput, - other: "FlowDataEngine", + other: FlowDataEngine, file_ref: str = None, flow_id: int = -1, node_id: int = -1, @@ -1710,9 +1712,9 @@ def fuzzy_join_external( def fuzzy_join( self, fuzzy_match_input: transform_schemas.FuzzyMatchInput, - other: "FlowDataEngine", + other: FlowDataEngine, node_logger: NodeLogger = None, - ) -> "FlowDataEngine": + ) -> FlowDataEngine: fuzzy_match_input_manager = transform_schemas.FuzzyMatchInputManager(fuzzy_match_input) left_df, right_df = prepare_for_fuzzy_match( left=self, right=other, fuzzy_match_input_manager=fuzzy_match_input_manager @@ -1729,8 +1731,8 @@ def do_cross_join( cross_join_input: transform_schemas.CrossJoinInput, auto_generate_selection: bool, verify_integrity: bool, - other: "FlowDataEngine", - ) -> "FlowDataEngine": + other: FlowDataEngine, + ) -> FlowDataEngine: """Performs a cross join with another DataFrame. A cross join produces the Cartesian product of the two DataFrames. @@ -1784,8 +1786,8 @@ def join( join_input: transform_schemas.JoinInput, auto_generate_selection: bool, verify_integrity: bool, - other: "FlowDataEngine", - ) -> "FlowDataEngine": + other: FlowDataEngine, + ) -> FlowDataEngine: """Performs a standard SQL-style join with another DataFrame.""" # Create manager from input join_manager = transform_schemas.JoinInputManager(join_input) @@ -1852,7 +1854,7 @@ def join( return FlowDataEngine(joined_df, calculate_schema_stats=False, number_of_records=0, streamable=False) - def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> "FlowDataEngine": + def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> FlowDataEngine: """Solves a graph problem represented by 'from' and 'to' columns. This is used for operations like finding connected components in a graph. @@ -1871,7 +1873,7 @@ def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> ) return FlowDataEngine(lf) - def add_new_values(self, values: Iterable, col_name: str = None) -> "FlowDataEngine": + def add_new_values(self, values: Iterable, col_name: str = None) -> FlowDataEngine: """Adds a new column with the provided values. Args: @@ -1885,7 +1887,7 @@ def add_new_values(self, values: Iterable, col_name: str = None) -> "FlowDataEng col_name = "new_values" return FlowDataEngine(self.data_frame.with_columns(pl.Series(values).alias(col_name))) - def get_record_count(self) -> "FlowDataEngine": + def get_record_count(self) -> FlowDataEngine: """Returns a new FlowDataEngine with a single column 'number_of_records' containing the total number of records. @@ -1894,7 +1896,7 @@ def get_record_count(self) -> "FlowDataEngine": """ return FlowDataEngine(self.data_frame.select(pl.len().alias("number_of_records"))) - def assert_equal(self, other: "FlowDataEngine", ordered: bool = True, strict_schema: bool = False): + def assert_equal(self, other: FlowDataEngine, ordered: bool = True, strict_schema: bool = False): """Asserts that this DataFrame is equal to another. Useful for testing. @@ -2063,7 +2065,7 @@ def get_select_inputs(self) -> transform_schemas.SelectInputs: [transform_schemas.SelectInput(old_name=c.name, data_type=c.data_type) for c in self.schema] ) - def select_columns(self, list_select: list[str] | tuple[str] | str) -> "FlowDataEngine": + def select_columns(self, list_select: list[str] | tuple[str] | str) -> FlowDataEngine: """Selects a subset of columns from the DataFrame. Args: @@ -2086,7 +2088,7 @@ def select_columns(self, list_select: list[str] | tuple[str] | str) -> "FlowData streamable=self._streamable, ) - def drop_columns(self, columns: list[str]) -> "FlowDataEngine": + def drop_columns(self, columns: list[str]) -> FlowDataEngine: """Drops specified columns from the DataFrame. Args: @@ -2103,7 +2105,7 @@ def drop_columns(self, columns: list[str]) -> "FlowDataEngine": self.data_frame.select(cols_for_select), number_of_records=self.number_of_records, schema=new_schema ) - def reorganize_order(self, column_order: list[str]) -> "FlowDataEngine": + def reorganize_order(self, column_order: list[str]) -> FlowDataEngine: """Reorganizes columns into a specified order. Args: @@ -2116,9 +2118,7 @@ def reorganize_order(self, column_order: list[str]) -> "FlowDataEngine": schema = sorted(self.schema, key=lambda x: column_order.index(x.column_name)) return FlowDataEngine(df, schema=schema, number_of_records=self.number_of_records) - def apply_flowfile_formula( - self, func: str, col_name: str, output_data_type: pl.DataType = None - ) -> "FlowDataEngine": + def apply_flowfile_formula(self, func: str, col_name: str, output_data_type: pl.DataType = None) -> FlowDataEngine: """Applies a formula to create a new column or transform an existing one. Args: @@ -2137,7 +2137,7 @@ def apply_flowfile_formula( return FlowDataEngine(df2, number_of_records=self.number_of_records) - def apply_sql_formula(self, func: str, col_name: str, output_data_type: pl.DataType = None) -> "FlowDataEngine": + def apply_sql_formula(self, func: str, col_name: str, output_data_type: pl.DataType = None) -> FlowDataEngine: """Applies an SQL-style formula using `pl.sql_expr`. Args: @@ -2158,7 +2158,7 @@ def apply_sql_formula(self, func: str, col_name: str, output_data_type: pl.DataT def output( self, output_fs: input_schema.OutputSettings, flow_id: int, node_id: int | str, execute_remote: bool = True - ) -> "FlowDataEngine": + ) -> FlowDataEngine: """Writes the DataFrame to an output file. Can execute the write operation locally or in a remote worker process. @@ -2202,7 +2202,7 @@ def output( logger.info("Finished writing output") return self - def make_unique(self, unique_input: transform_schemas.UniqueInput = None) -> "FlowDataEngine": + def make_unique(self, unique_input: transform_schemas.UniqueInput = None) -> FlowDataEngine: """Gets the unique rows from the DataFrame. Args: @@ -2216,7 +2216,7 @@ def make_unique(self, unique_input: transform_schemas.UniqueInput = None) -> "Fl return FlowDataEngine(self.data_frame.unique()) return FlowDataEngine(self.data_frame.unique(unique_input.columns, keep=unique_input.strategy)) - def concat(self, other: Iterable["FlowDataEngine"] | "FlowDataEngine") -> "FlowDataEngine": + def concat(self, other: Iterable[FlowDataEngine] | FlowDataEngine) -> FlowDataEngine: """Concatenates this DataFrame with one or more other DataFrames. Args: @@ -2231,7 +2231,7 @@ def concat(self, other: Iterable["FlowDataEngine"] | "FlowDataEngine") -> "FlowD dfs: list[pl.LazyFrame] | list[pl.DataFrame] = [self.data_frame] + [flt.data_frame for flt in other] return FlowDataEngine(pl.concat(dfs, how="diagonal_relaxed")) - def do_select(self, select_inputs: transform_schemas.SelectInputs, keep_missing: bool = True) -> "FlowDataEngine": + def do_select(self, select_inputs: transform_schemas.SelectInputs, keep_missing: bool = True) -> FlowDataEngine: """Performs a complex column selection, renaming, and reordering operation. Args: @@ -2310,7 +2310,7 @@ def create_from_path_worker(cls, received_table: input_schema.ReceivedTable, flo return cls(external_fetcher.get_result()) -def execute_polars_code(*flowfile_tables: "FlowDataEngine", code: str) -> "FlowDataEngine": +def execute_polars_code(*flowfile_tables: FlowDataEngine, code: str) -> FlowDataEngine: """Executes arbitrary Polars code on one or more FlowDataEngine objects. This function takes a string of Python code that uses Polars and executes it. diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py index d8002d79a..20821e26e 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py @@ -268,6 +268,6 @@ def assert_if_flowfile_schema(obj: Iterable) -> bool: """ Assert that the object is a valid iterable of FlowfileColumn objects. """ - if isinstance(obj, (list, set, tuple)): + if isinstance(obj, list | set | tuple): return all(isinstance(item, FlowfileColumn) for item in obj) return False diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/polars_code_parser.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/polars_code_parser.py index 04c77c3a7..9e83f2072 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/polars_code_parser.py @@ -114,7 +114,7 @@ def visit_AsyncFunctionDef(self, node): def visit_Expr(self, node): # Remove standalone string literals - if isinstance(node.value, (ast.Str, ast.Constant)) and isinstance(getattr(node.value, "value", None), str): + if isinstance(node.value, ast.Str | ast.Constant) and isinstance(getattr(node.value, "value", None), str): return None return self.generic_visit(node) @@ -202,7 +202,7 @@ def _validate_code(code: str) -> None: tree = ast.parse(code) for node in ast.walk(tree): # Block imports - if isinstance(node, (ast.Import, ast.ImportFrom)): + if isinstance(node, ast.Import | ast.ImportFrom): raise ValueError("Import statements are not allowed") # Block exec/eval diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/read_excel_tables.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/read_excel_tables.py index ef6cc3f5b..50835d01e 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/read_excel_tables.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/read_excel_tables.py @@ -21,8 +21,7 @@ def raw_data_openpyxl( workbook: Workbook = load_workbook(file_path, data_only=True, read_only=True) sheet_name = workbook.sheetnames[0] if sheet_name is None else sheet_name sheet: Worksheet = workbook[sheet_name] - for row in sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True): - yield row + yield from sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True) workbook.close() del workbook gc.collect() diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/sample_data.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/sample_data.py index 889ad4366..662deb4e5 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/sample_data.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/sample_data.py @@ -49,7 +49,7 @@ def generate_phone_number(): return fake.phone_number() data = [] - for i in range(max_n_records): + for _i in range(max_n_records): name = generate_name() data.append( dict( @@ -92,7 +92,9 @@ def create_fake_data_raw( first_names = partial(fake.first_name) last_names = partial(fake.last_name) domain_names = [fake.domain_name() for _ in range(10)] # Pre-generate a small list - sales_data = lambda: fake.random_int(0, 1000) + + def sales_data(): + return fake.random_int(0, 1000) def generate_name(): return f"{first_names()} {last_names()}" diff --git a/flowfile_core/flowfile_core/flowfile/flow_graph.py b/flowfile_core/flowfile_core/flowfile/flow_graph.py index d22b9e079..76dc21cac 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_graph.py +++ b/flowfile_core/flowfile_core/flowfile/flow_graph.py @@ -80,7 +80,7 @@ def represent_list_json(dumper, data): """Use inline style for short simple lists, block style for complex ones.""" - if len(data) <= 10 and all(isinstance(item, (int, str, float, bool, type(None))) for item in data): + if len(data) <= 10 and all(isinstance(item, int | str | float | bool | type(None)) for item in data): return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=True) return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=False) @@ -564,7 +564,7 @@ def _func(*flow_data_engine: FlowDataEngine) -> FlowDataEngine | None: accessed_secrets = custom_node.get_accessed_secrets() if accessed_secrets: logger.info(f"Node '{user_defined_node_settings.node_id}' accessed secrets: {accessed_secrets}") - if isinstance(output, (pl.LazyFrame, pl.DataFrame)): + if isinstance(output, pl.LazyFrame | pl.DataFrame): return FlowDataEngine(output) return None @@ -2348,7 +2348,7 @@ def get_frontend_data(self) -> dict: for o in node_info.outputs: outputs[o] += 1 connections = [] - for output_node_id, n_connections in outputs.items(): + for output_node_id, _n_connections in outputs.items(): leading_to_node = self.get_node(output_node_id) input_types = leading_to_node.get_input_type(node_info.id) for input_type in input_types: diff --git a/flowfile_core/flowfile_core/flowfile/flow_node/flow_node.py b/flowfile_core/flowfile_core/flowfile/flow_node/flow_node.py index 3f2b45c98..e0cb67559 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_node/flow_node.py +++ b/flowfile_core/flowfile_core/flowfile/flow_node/flow_node.py @@ -665,8 +665,7 @@ def get_all_dependent_nodes(self) -> Generator["FlowNode", None, None]: """ for node in self.leads_to_nodes: yield node - for n in node.get_all_dependent_nodes(): - yield n + yield from node.get_all_dependent_nodes() def get_all_dependent_node_ids(self) -> Generator[int, None, None]: """Yields the IDs of all downstream nodes recursively. @@ -676,8 +675,7 @@ def get_all_dependent_node_ids(self) -> Generator[int, None, None]: """ for node in self.leads_to_nodes: yield node.node_id - for n in node.get_all_dependent_node_ids(): - yield n + yield from node.get_all_dependent_node_ids() @property def schema(self) -> list[FlowfileColumn]: diff --git a/flowfile_core/flowfile_core/flowfile/handler.py b/flowfile_core/flowfile_core/flowfile/handler.py index d23e75d18..c4847b9b0 100644 --- a/flowfile_core/flowfile_core/flowfile/handler.py +++ b/flowfile_core/flowfile_core/flowfile/handler.py @@ -134,10 +134,7 @@ def add_flow(self, name: str = None, flow_path: str = None, user_id: int | None name = create_flow_name() if not flow_path: flow_path = get_flow_save_location(name) - flow_info = FlowSettings( - name=name, flow_id=next_id, save_location=str(flow_path), - path=str(flow_path) - ) + flow_info = FlowSettings(name=name, flow_id=next_id, save_location=str(flow_path), path=str(flow_path)) flow = self.register_flow(flow_info, user_id=user_id) flow.save_flow(flow.flow_settings.path) return next_id diff --git a/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py b/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py index 1e09e139b..3f71effc5 100644 --- a/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py +++ b/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py @@ -70,7 +70,7 @@ def parse_schema(schema: list[Any]) -> list[FlowfileColumn]: first_col = schema[0] if isinstance(first_col, dict): return [FlowfileColumn(**col) for col in schema] - elif isinstance(first_col, (list, tuple)): + elif isinstance(first_col, list | tuple): return [FlowfileColumn.from_input(column_name=col[0], data_type=col[1]) for col in schema] elif isinstance(first_col, str): return [FlowfileColumn.from_input(column_name=col, data_type="varchar") for col in schema] @@ -99,7 +99,7 @@ def get_iter(self) -> Generator[dict[str, Any], None, None]: def get_sample(self, n: int = 10000): data = self.get_iter() - for i in range(n): + for _i in range(n): try: yield next(data) except StopIteration: diff --git a/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py b/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py index 794233cca..b5875ff6b 100644 --- a/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py +++ b/flowfile_core/flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py @@ -18,7 +18,7 @@ def getter(data: SampleUsers) -> Generator[dict[str, Any], None, None]: """ index_pos = 0 - for i in range(data.size): + for _i in range(data.size): sleep(0.01) headers = {"x-api-key": "reqres-free-v1"} diff --git a/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py b/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py index c6c7243af..feec6e906 100644 --- a/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py +++ b/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py @@ -136,7 +136,13 @@ def get_query_columns(engine: Engine, query_text: str): Returns: Dictionary mapping column names to string type + + Raises: + UnsafeSQLError: If the query contains unsafe operations """ + # Validate the query for safety before execution + validate_sql_query(query_text) + with engine.connect() as connection: # Create a text object from the query query = text(query_text) @@ -302,8 +308,7 @@ def validate(self) -> None: def get_iter(self) -> Generator[dict[str, Any], None, None]: logger.warning("Getting data in iteration, this is suboptimal") data = self.data_getter() - for row in data: - yield row + yield from data def get_df(self): df = self.get_pl_df() diff --git a/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/utils.py b/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/utils.py index e70ce394b..251d66757 100644 --- a/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/utils.py +++ b/flowfile_core/flowfile_core/flowfile/sources/external_sources/sql_source/utils.py @@ -232,7 +232,6 @@ "tinyint unsigned": pl.UInt8, "mediumint unsigned": pl.UInt32, "year": pl.Int16, - # --- Floats & Decimals --- "numeric": pl.Decimal, "decimal": pl.Decimal, @@ -247,12 +246,10 @@ "double precision": pl.Float64, "binary_float": pl.Float32, # Oracle "binary_double": pl.Float64, # Oracle - # --- Booleans --- "boolean": pl.Boolean, "bool": pl.Boolean, "bit": pl.Boolean, # Note: PostgreSQL 'bit' is varying, but MSSQL/MySQL 'bit' is boolean. Defaulting to Bool. - # --- Strings / Text --- "varchar": pl.Utf8, "varchar2": pl.Utf8, # Oracle @@ -279,7 +276,6 @@ "xmltype": pl.Utf8, "json": pl.Utf8, "jsonb": pl.Utf8, - # --- Network / Specialized Strings (Postgres) --- "uuid": pl.Utf8, "cidr": pl.Utf8, @@ -292,7 +288,6 @@ "geography": pl.Utf8, "hierarchyid": pl.Utf8, "bit varying": pl.Utf8, - # --- Dates & Times --- "date": pl.Date, "datetime": pl.Datetime, @@ -306,12 +301,10 @@ "time": pl.Time, "time without time zone": pl.Time, "time with time zone": pl.Time, - # --- Durations / Intervals --- "interval": pl.Duration, "interval year to month": pl.Duration, # Oracle "interval day to second": pl.Duration, # Oracle - # --- Binary --- "bytea": pl.Binary, # Postgres "binary": pl.Binary, @@ -324,7 +317,6 @@ "long raw": pl.Binary, # Oracle "bfile": pl.Binary, # Oracle "image": pl.Binary, # MSSQL - # --- Other --- "null": None, "sql_variant": pl.Object, diff --git a/flowfile_core/flowfile_core/flowfile/util/calculate_layout.py b/flowfile_core/flowfile_core/flowfile/util/calculate_layout.py index 5e2b7b187..ecab2a74b 100644 --- a/flowfile_core/flowfile_core/flowfile/util/calculate_layout.py +++ b/flowfile_core/flowfile_core/flowfile/util/calculate_layout.py @@ -70,7 +70,7 @@ def calculate_layered_layout( current_in_degree = defaultdict(int, in_degree) while queue: - stage_size = len(queue) + len(queue) processing_order = sorted(list(queue)) queue.clear() nodes_in_current_stage = [] diff --git a/flowfile_core/flowfile_core/routes/auth.py b/flowfile_core/flowfile_core/routes/auth.py index 5594f11ea..3c1995bfe 100644 --- a/flowfile_core/flowfile_core/routes/auth.py +++ b/flowfile_core/flowfile_core/routes/auth.py @@ -1,16 +1,15 @@ # app_routes/auth.py import os -from typing import Optional, List -from fastapi import APIRouter, Depends, HTTPException, status, Request, Form +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status from sqlalchemy.orm import Session -from flowfile_core.auth.jwt import get_current_active_user, get_current_admin_user, create_access_token -from flowfile_core.auth.models import Token, User, UserCreate, UserUpdate, ChangePassword -from flowfile_core.auth.password import verify_password, get_password_hash, validate_password, PASSWORD_REQUIREMENTS -from flowfile_core.database.connection import get_db +from flowfile_core.auth.jwt import create_access_token, get_current_active_user, get_current_admin_user +from flowfile_core.auth.models import ChangePassword, Token, User, UserCreate, UserUpdate +from flowfile_core.auth.password import PASSWORD_REQUIREMENTS, get_password_hash, validate_password, verify_password from flowfile_core.database import models as db_models +from flowfile_core.database.connection import get_db router = APIRouter() @@ -19,8 +18,8 @@ async def login_for_access_token( request: Request, db: Session = Depends(get_db), - username: Optional[str] = Form(None), - password: Optional[str] = Form(None) + username: str | None = Form(None), + password: str | None = Form(None), ): # In Electron mode, auto-authenticate without requiring form data if os.environ.get("FLOWFILE_MODE") == "electron": @@ -35,9 +34,7 @@ async def login_for_access_token( headers={"WWW-Authenticate": "Bearer"}, ) - user = db.query(db_models.User).filter( - db_models.User.username == username - ).first() + user = db.query(db_models.User).filter(db_models.User.username == username).first() if not user or not verify_password(password, user.hashed_password): raise HTTPException( @@ -58,11 +55,9 @@ async def read_users_me(current_user=Depends(get_current_active_user)): # ============= Admin User Management Endpoints ============= -@router.get("/users", response_model=List[User]) -async def list_users( - current_user: User = Depends(get_current_admin_user), - db: Session = Depends(get_db) -): + +@router.get("/users", response_model=list[User]) +async def list_users(current_user: User = Depends(get_current_admin_user), db: Session = Depends(get_db)): """List all users (admin only)""" users = db.query(db_models.User).all() return [ @@ -73,7 +68,7 @@ async def list_users( full_name=u.full_name, disabled=u.disabled, is_admin=u.is_admin, - must_change_password=u.must_change_password + must_change_password=u.must_change_password, ) for u in users ] @@ -81,39 +76,24 @@ async def list_users( @router.post("/users", response_model=User) async def create_user( - user_data: UserCreate, - current_user: User = Depends(get_current_admin_user), - db: Session = Depends(get_db) + user_data: UserCreate, current_user: User = Depends(get_current_admin_user), db: Session = Depends(get_db) ): """Create a new user (admin only)""" # Check if username already exists - existing_user = db.query(db_models.User).filter( - db_models.User.username == user_data.username - ).first() + existing_user = db.query(db_models.User).filter(db_models.User.username == user_data.username).first() if existing_user: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Username already exists" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Username already exists") # Check if email already exists (if provided) if user_data.email: - existing_email = db.query(db_models.User).filter( - db_models.User.email == user_data.email - ).first() + existing_email = db.query(db_models.User).filter(db_models.User.email == user_data.email).first() if existing_email: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Email already exists" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email already exists") # Validate password requirements is_valid, error_message = validate_password(user_data.password) if not is_valid: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=error_message - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message) # Create new user with must_change_password=True hashed_password = get_password_hash(user_data.password) @@ -123,7 +103,7 @@ async def create_user( full_name=user_data.full_name, hashed_password=hashed_password, is_admin=user_data.is_admin, - must_change_password=True + must_change_password=True, ) db.add(new_user) db.commit() @@ -136,7 +116,7 @@ async def create_user( full_name=new_user.full_name, disabled=new_user.disabled, is_admin=new_user.is_admin, - must_change_password=new_user.must_change_password + must_change_password=new_user.must_change_password, ) @@ -145,42 +125,31 @@ async def update_user( user_id: int, user_data: UserUpdate, current_user: User = Depends(get_current_admin_user), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ): """Update a user (admin only)""" user = db.query(db_models.User).filter(db_models.User.id == user_id).first() if not user: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found" - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found") # Prevent admin from disabling themselves if user.id == current_user.id and user_data.disabled: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Cannot disable your own account" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot disable your own account") # Prevent admin from removing their own admin status if user.id == current_user.id and user_data.is_admin is False: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Cannot remove your own admin privileges" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot remove your own admin privileges") # Update fields if user_data.email is not None: # Check if email already exists for another user - existing_email = db.query(db_models.User).filter( - db_models.User.email == user_data.email, - db_models.User.id != user_id - ).first() + existing_email = ( + db.query(db_models.User) + .filter(db_models.User.email == user_data.email, db_models.User.id != user_id) + .first() + ) if existing_email: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Email already exists" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email already exists") user.email = user_data.email if user_data.full_name is not None: @@ -196,10 +165,7 @@ async def update_user( # Validate password requirements is_valid, error_message = validate_password(user_data.password) if not is_valid: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=error_message - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message) user.hashed_password = get_password_hash(user_data.password) # Reset must_change_password when admin sets a new password user.must_change_password = True @@ -217,30 +183,22 @@ async def update_user( full_name=user.full_name, disabled=user.disabled, is_admin=user.is_admin, - must_change_password=user.must_change_password + must_change_password=user.must_change_password, ) @router.delete("/users/{user_id}") async def delete_user( - user_id: int, - current_user: User = Depends(get_current_admin_user), - db: Session = Depends(get_db) + user_id: int, current_user: User = Depends(get_current_admin_user), db: Session = Depends(get_db) ): """Delete a user (admin only)""" user = db.query(db_models.User).filter(db_models.User.id == user_id).first() if not user: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found" - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found") # Prevent admin from deleting themselves if user.id == current_user.id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Cannot delete your own account" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot delete your own account") # Delete user's secrets and connections first (cascade) db.query(db_models.Secret).filter(db_models.Secret.user_id == user_id).delete() @@ -255,34 +213,24 @@ async def delete_user( # ============= User Self-Service Endpoints ============= + @router.post("/users/me/change-password", response_model=User) async def change_own_password( - password_data: ChangePassword, - current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + password_data: ChangePassword, current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db) ): """Change the current user's password""" user = db.query(db_models.User).filter(db_models.User.id == current_user.id).first() if not user: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found" - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found") # Verify current password if not verify_password(password_data.current_password, user.hashed_password): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Current password is incorrect" - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Current password is incorrect") # Validate new password requirements is_valid, error_message = validate_password(password_data.new_password) if not is_valid: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=error_message - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message) # Update password and clear must_change_password flag user.hashed_password = get_password_hash(password_data.new_password) @@ -297,7 +245,7 @@ async def change_own_password( full_name=user.full_name, disabled=user.disabled, is_admin=user.is_admin, - must_change_password=user.must_change_password + must_change_password=user.must_change_password, ) diff --git a/flowfile_core/flowfile_core/routes/routes.py b/flowfile_core/flowfile_core/routes/routes.py index a6f44fd86..c38d9e136 100644 --- a/flowfile_core/flowfile_core/routes/routes.py +++ b/flowfile_core/flowfile_core/routes/routes.py @@ -11,52 +11,52 @@ import logging import os from pathlib import Path -from typing import List, Dict, Any, Optional +from typing import Any -from fastapi import APIRouter, File, UploadFile, BackgroundTasks, HTTPException, status, Body, Depends +from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, UploadFile, status from fastapi.responses import JSONResponse, Response + # External dependencies from polars_expr_transformer.function_overview import get_all_expressions, get_expression_overview from sqlalchemy.orm import Session from flowfile_core import flow_file_handler + # Core modules from flowfile_core.auth.jwt import get_current_active_user from flowfile_core.configs import logger -from flowfile_core.configs.node_store import nodes_list, check_if_has_default_setting +from flowfile_core.configs.node_store import check_if_has_default_setting, nodes_list from flowfile_core.database.connection import get_db + # File handling from flowfile_core.fileExplorer.funcs import ( - SecureFileExplorer, FileInfo, + SecureFileExplorer, get_files_from_directory, - validate_file_path, validate_path_under_cwd, ) from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars -from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection, - get_database_connection, - delete_database_connection, - get_all_database_connections_interface) +from flowfile_core.flowfile.database_connection_manager.db_connections import ( + delete_database_connection, + get_all_database_connections_interface, + get_database_connection, + store_database_connection, +) from flowfile_core.flowfile.extensions import get_instant_func_results from flowfile_core.flowfile.flow_graph import add_connection, delete_connection from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings from flowfile_core.run_lock import get_flow_run_lock -from flowfile_core.schemas import input_schema, schemas, output_model +from flowfile_core.schemas import input_schema, output_model, schemas from flowfile_core.utils import excel_file_manager from flowfile_core.utils.fileManager import create_dir from flowfile_core.utils.utils import camel_case_to_snake_case from shared.storage_config import storage - router = APIRouter(dependencies=[Depends(get_current_active_user)]) # Initialize services -file_explorer = SecureFileExplorer( - start_path=storage.user_data_directory, - sandbox_root=storage.user_data_directory -) +file_explorer = SecureFileExplorer(start_path=storage.user_data_directory, sandbox_root=storage.user_data_directory) def get_node_model(setting_name_ref: str): @@ -80,7 +80,7 @@ async def upload_file(file: UploadFile = File(...)) -> JSONResponse: """ safe_name = Path(file.filename).name.replace("..", "") if not safe_name: - raise HTTPException(400, 'Invalid filename') + raise HTTPException(400, "Invalid filename") uploads_dir = Path("uploads") uploads_dir.mkdir(exist_ok=True) file_location = uploads_dir / safe_name @@ -89,7 +89,7 @@ async def upload_file(file: UploadFile = File(...)) -> JSONResponse: return JSONResponse(content={"filename": safe_name, "filepath": str(file_location)}) -@router.get('/files/files_in_local_directory/', response_model=list[FileInfo], tags=['file manager']) +@router.get("/files/files_in_local_directory/", response_model=list[FileInfo], tags=["file manager"]) async def get_local_files(directory: str) -> list[FileInfo]: """Retrieves a list of files from a specified local directory. @@ -104,58 +104,56 @@ async def get_local_files(directory: str) -> list[FileInfo]: HTTPException: 403 if access is denied (path outside sandbox). """ # Validate path is within sandbox before proceeding - explorer = SecureFileExplorer( - start_path=storage.user_data_directory, - sandbox_root=storage.user_data_directory - ) + explorer = SecureFileExplorer(start_path=storage.user_data_directory, sandbox_root=storage.user_data_directory) validated_path = explorer.get_absolute_path(directory) if validated_path is None: - raise HTTPException(403, 'Access denied or directory does not exist') + raise HTTPException(403, "Access denied or directory does not exist") if not validated_path.exists() or not validated_path.is_dir(): - raise HTTPException(404, 'Directory does not exist') + raise HTTPException(404, "Directory does not exist") files = get_files_from_directory(str(validated_path), sandbox_root=storage.user_data_directory) if files is None: - raise HTTPException(403, 'Access denied or directory does not exist') + raise HTTPException(403, "Access denied or directory does not exist") return files -@router.get('/files/tree/', response_model=List[FileInfo], tags=['file manager']) -async def get_current_files() -> List[FileInfo]: +@router.get("/files/tree/", response_model=list[FileInfo], tags=["file manager"]) +async def get_current_files() -> list[FileInfo]: """Gets the contents of the file explorer's current directory.""" f = file_explorer.list_contents() return f -@router.post('/files/navigate_up/', response_model=str, tags=['file manager']) +@router.post("/files/navigate_up/", response_model=str, tags=["file manager"]) async def navigate_up() -> str: """Navigates the file explorer one directory level up.""" file_explorer.navigate_up() return str(file_explorer.current_path) -@router.post('/files/navigate_into/', response_model=str, tags=['file manager']) +@router.post("/files/navigate_into/", response_model=str, tags=["file manager"]) async def navigate_into_directory(directory_name: str) -> str: """Navigates the file explorer into a specified subdirectory.""" file_explorer.navigate_into(directory_name) return str(file_explorer.current_path) -@router.post('/files/navigate_to/', tags=['file manager']) +@router.post("/files/navigate_to/", tags=["file manager"]) async def navigate_to_directory(directory_name: str) -> str: """Navigates the file explorer to an absolute directory path.""" file_explorer.navigate_to(directory_name) return str(file_explorer.current_path) -@router.get('/files/current_path/', response_model=str, tags=['file manager']) +@router.get("/files/current_path/", response_model=str, tags=["file manager"]) async def get_current_path() -> str: """Returns the current absolute path of the file explorer.""" return str(file_explorer.current_path) -@router.get('/files/directory_contents/', response_model=List[FileInfo], tags=['file manager']) -async def get_directory_contents(directory: str, file_types: List[str] = None, - include_hidden: bool = False) -> List[FileInfo]: +@router.get("/files/directory_contents/", response_model=list[FileInfo], tags=["file manager"]) +async def get_directory_contents( + directory: str, file_types: list[str] = None, include_hidden: bool = False +) -> list[FileInfo]: """Gets the contents of an arbitrary directory path. Args: @@ -171,16 +169,16 @@ async def get_directory_contents(directory: str, file_types: List[str] = None, return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types) except Exception as e: logger.error(e) - HTTPException(404, 'Could not access the directory') + HTTPException(404, "Could not access the directory") -@router.get('/files/current_directory_contents/', response_model=List[FileInfo], tags=['file manager']) -async def get_current_directory_contents(file_types: List[str] = None, include_hidden: bool = False) -> List[FileInfo]: +@router.get("/files/current_directory_contents/", response_model=list[FileInfo], tags=["file manager"]) +async def get_current_directory_contents(file_types: list[str] = None, include_hidden: bool = False) -> list[FileInfo]: """Gets the contents of the file explorer's current directory.""" return file_explorer.list_contents(file_types=file_types, show_hidden=include_hidden) -@router.post('/files/create_directory', response_model=output_model.OutputDir, tags=['file manager']) +@router.post("/files/create_directory", response_model=output_model.OutputDir, tags=["file manager"]) def create_directory(new_directory: input_schema.NewDirectory) -> bool: """Creates a new directory at the specified path. @@ -218,25 +216,25 @@ async def get_active_flow_file_sessions(current_user=Depends(get_current_active_ return [flf.flow_settings for flf in flow_file_handler.get_user_flows(user_id)] -@router.post("/node/trigger_fetch_data", tags=['editor']) +@router.post("/node/trigger_fetch_data", tags=["editor"]) async def trigger_fetch_node_data(flow_id: int, node_id: int, background_tasks: BackgroundTasks): """Fetches and refreshes the data for a specific node.""" flow = flow_file_handler.get_flow(flow_id) lock = get_flow_run_lock(flow_id) async with lock: if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is already running') + raise HTTPException(422, "Flow is already running") try: flow.validate_if_node_can_be_fetched(node_id) except Exception as e: raise HTTPException(422, str(e)) background_tasks.add_task(flow.trigger_fetch_node, node_id) - return JSONResponse(content={"message": "Data started", - "flow_id": flow_id, - "node_id": node_id}, status_code=status.HTTP_200_OK) + return JSONResponse( + content={"message": "Data started", "flow_id": flow_id, "node_id": node_id}, status_code=status.HTTP_200_OK + ) -@router.post('/flow/run/', tags=['editor']) +@router.post("/flow/run/", tags=["editor"]) async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse: """Executes a flow in a background task. @@ -247,22 +245,22 @@ async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONRespo Returns: A JSON response indicating that the flow has started. """ - logger.info('starting to run...') + logger.info("starting to run...") flow = flow_file_handler.get_flow(flow_id) lock = get_flow_run_lock(flow_id) async with lock: if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is already running') + raise HTTPException(422, "Flow is already running") background_tasks.add_task(flow.run_graph) return JSONResponse(content={"message": "Data started", "flow_id": flow_id}, status_code=status.HTTP_200_OK) -@router.post('/flow/cancel/', tags=['editor']) +@router.post("/flow/cancel/", tags=["editor"]) def cancel_flow(flow_id: int): """Cancels a currently running flow execution.""" flow = flow_file_handler.get_flow(flow_id) if not flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is not running') + raise HTTPException(422, "Flow is not running") flow.cancel() @@ -276,8 +274,7 @@ def apply_standard_layout(flow_id: int): flow.apply_layout() -@router.get('/flow/run_status/', tags=['editor'], - response_model=output_model.RunInformation) +@router.get("/flow/run_status/", tags=["editor"], response_model=output_model.RunInformation) def get_run_status(flow_id: int, response: Response): """Retrieves the run status information for a specific flow. @@ -293,23 +290,23 @@ def get_run_status(flow_id: int, response: Response): return flow.get_run_info() -@router.post('/transform/manual_input', tags=['transform']) +@router.post("/transform/manual_input", tags=["transform"]) def add_manual_input(manual_input: input_schema.NodeManualInput): flow = flow_file_handler.get_flow(manual_input.flow_id) flow.add_datasource(manual_input) -@router.post('/transform/add_input/', tags=['transform']) +@router.post("/transform/add_input/", tags=["transform"]) def add_flow_input(input_data: input_schema.NodeDatasource): flow = flow_file_handler.get_flow(input_data.flow_id) try: flow.add_datasource(input_data) except: - input_data.file_ref = os.path.join('db_data', input_data.file_ref) + input_data.file_ref = os.path.join("db_data", input_data.file_ref) flow.add_datasource(input_data) -@router.post('/editor/copy_node', tags=['editor']) +@router.post("/editor/copy_node", tags=["editor"]) def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise: input_schema.NodePromise): """Copies an existing node's settings to a new node promise. @@ -320,10 +317,11 @@ def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise """ try: flow_to_copy_from = flow_file_handler.get_flow(flow_id_to_copy_from) - flow = (flow_to_copy_from - if flow_id_to_copy_from == node_promise.flow_id - else flow_file_handler.get_flow(node_promise.flow_id) - ) + flow = ( + flow_to_copy_from + if flow_id_to_copy_from == node_promise.flow_id + else flow_file_handler.get_flow(node_promise.flow_id) + ) node_to_copy = flow_to_copy_from.get_node(node_id_to_copy_from) logger.info(f"Copying data {node_promise.node_type}") @@ -344,7 +342,7 @@ def copy_node(node_id_to_copy_from: int, flow_id_to_copy_from: int, node_promise raise HTTPException(422, str(e)) -@router.post('/editor/add_node/', tags=['editor']) +@router.post("/editor/add_node/", tags=["editor"]) def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y: int = 0): """Adds a new, unconfigured node (a "promise") to the flow graph. @@ -356,16 +354,16 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y: pos_y: The Y coordinate for the node's position in the UI. """ flow = flow_file_handler.get_flow(flow_id) - logger.info(f'Adding a promise for {node_type}') + logger.info(f"Adding a promise for {node_type}") if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is running') + raise HTTPException(422, "Flow is running") node = flow.get_node(node_id) if node is not None: flow.delete_node(node_id) - node_promise = input_schema.NodePromise(flow_id=flow_id, node_id=node_id, cache_results=False, pos_x=pos_x, - pos_y=pos_y, - node_type=node_type) - if node_type == 'explore_data': + node_promise = input_schema.NodePromise( + flow_id=flow_id, node_id=node_id, cache_results=False, pos_x=pos_x, pos_y=pos_y, node_type=node_type + ) + if node_type == "explore_data": flow.add_initial_node_analysis(node_promise) return else: @@ -373,102 +371,105 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y: flow.add_node_promise(node_promise) if check_if_has_default_setting(node_type): - logger.info(f'Found standard settings for {node_type}, trying to upload them') - setting_name_ref = 'node' + node_type.replace('_', '') + logger.info(f"Found standard settings for {node_type}, trying to upload them") + setting_name_ref = "node" + node_type.replace("_", "") node_model = get_node_model(setting_name_ref) - add_func = getattr(flow, 'add_' + node_type) - initial_settings = node_model(flow_id=flow_id, node_id=node_id, cache_results=False, - pos_x=pos_x, pos_y=pos_y, node_type=node_type) + add_func = getattr(flow, "add_" + node_type) + initial_settings = node_model( + flow_id=flow_id, node_id=node_id, cache_results=False, pos_x=pos_x, pos_y=pos_y, node_type=node_type + ) add_func(initial_settings) -@router.post('/editor/delete_node/', tags=['editor']) -def delete_node(flow_id: Optional[int], node_id: int): +@router.post("/editor/delete_node/", tags=["editor"]) +def delete_node(flow_id: int | None, node_id: int): """Deletes a node from the flow graph.""" - logger.info('Deleting node') + logger.info("Deleting node") flow = flow_file_handler.get_flow(flow_id) if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is running') + raise HTTPException(422, "Flow is running") flow.delete_node(node_id) -@router.post('/editor/delete_connection/', tags=['editor']) +@router.post("/editor/delete_connection/", tags=["editor"]) def delete_node_connection(flow_id: int, node_connection: input_schema.NodeConnection = None): """Deletes a connection (edge) between two nodes.""" flow_id = int(flow_id) logger.info( - f'Deleting connection node {node_connection.output_connection.node_id} to node {node_connection.input_connection.node_id}') + f"Deleting connection node {node_connection.output_connection.node_id} to node {node_connection.input_connection.node_id}" + ) flow = flow_file_handler.get_flow(flow_id) if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is running') + raise HTTPException(422, "Flow is running") delete_connection(flow, node_connection) -@router.post("/db_connection_lib", tags=['db_connections']) -def create_db_connection(input_connection: input_schema.FullDatabaseConnection, - current_user=Depends(get_current_active_user), - db: Session = Depends(get_db) - ): +@router.post("/db_connection_lib", tags=["db_connections"]) +def create_db_connection( + input_connection: input_schema.FullDatabaseConnection, + current_user=Depends(get_current_active_user), + db: Session = Depends(get_db), +): """Creates and securely stores a new database connection.""" - logger.info(f'Creating database connection {input_connection.connection_name}') + logger.info(f"Creating database connection {input_connection.connection_name}") try: store_database_connection(db, input_connection, current_user.id) except ValueError: - raise HTTPException(422, 'Connection name already exists') + raise HTTPException(422, "Connection name already exists") except Exception as e: logger.error(e) raise HTTPException(422, str(e)) return {"message": "Database connection created successfully"} -@router.delete('/db_connection_lib', tags=['db_connections']) -def delete_db_connection(connection_name: str, - current_user=Depends(get_current_active_user), - db: Session = Depends(get_db) - ): +@router.delete("/db_connection_lib", tags=["db_connections"]) +def delete_db_connection( + connection_name: str, current_user=Depends(get_current_active_user), db: Session = Depends(get_db) +): """Deletes a stored database connection.""" - logger.info(f'Deleting database connection {connection_name}') + logger.info(f"Deleting database connection {connection_name}") db_connection = get_database_connection(db, connection_name, current_user.id) if db_connection is None: - raise HTTPException(404, 'Database connection not found') + raise HTTPException(404, "Database connection not found") delete_database_connection(db, connection_name, current_user.id) return {"message": "Database connection deleted successfully"} -@router.get('/db_connection_lib', tags=['db_connections'], - response_model=List[input_schema.FullDatabaseConnectionInterface]) +@router.get( + "/db_connection_lib", tags=["db_connections"], response_model=list[input_schema.FullDatabaseConnectionInterface] +) def get_db_connections( - db: Session = Depends(get_db), - current_user=Depends(get_current_active_user)) -> List[input_schema.FullDatabaseConnectionInterface]: + db: Session = Depends(get_db), current_user=Depends(get_current_active_user) +) -> list[input_schema.FullDatabaseConnectionInterface]: """Retrieves all stored database connections for the current user (without passwords).""" return get_all_database_connections_interface(db, current_user.id) -@router.post('/editor/connect_node/', tags=['editor']) +@router.post("/editor/connect_node/", tags=["editor"]) def connect_node(flow_id: int, node_connection: input_schema.NodeConnection): """Creates a connection (edge) between two nodes in the flow graph.""" flow = flow_file_handler.get_flow(flow_id) if flow is None: - logger.info('could not find the flow') - raise HTTPException(404, 'could not find the flow') + logger.info("could not find the flow") + raise HTTPException(404, "could not find the flow") if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is running') + raise HTTPException(422, "Flow is running") add_connection(flow, node_connection) -@router.get('/editor/expression_doc', tags=['editor'], response_model=List[output_model.ExpressionsOverview]) -def get_expression_doc() -> List[output_model.ExpressionsOverview]: +@router.get("/editor/expression_doc", tags=["editor"], response_model=list[output_model.ExpressionsOverview]) +def get_expression_doc() -> list[output_model.ExpressionsOverview]: """Retrieves documentation for available Polars expressions.""" return get_expression_overview() -@router.get('/editor/expressions', tags=['editor'], response_model=List[str]) -def get_expressions() -> List[str]: +@router.get("/editor/expressions", tags=["editor"], response_model=list[str]) +def get_expressions() -> list[str]: """Retrieves a list of all available Flowfile expression names.""" return get_all_expressions() -@router.get('/editor/flow', tags=['editor'], response_model=schemas.FlowSettings) +@router.get("/editor/flow", tags=["editor"], response_model=schemas.FlowSettings) def get_flow(flow_id: int): """Retrieves the settings for a specific flow.""" flow_id = int(flow_id) @@ -482,7 +483,7 @@ def get_generated_code(flow_id: int) -> str: flow_id = int(flow_id) flow = flow_file_handler.get_flow(flow_id) if flow is None: - raise HTTPException(404, 'could not find the flow') + raise HTTPException(404, "could not find the flow") return export_flow_to_polars(flow) @@ -493,7 +494,7 @@ def create_flow(flow_path: str = None, name: str = None, current_user=Depends(ge name = Path(flow_path).stem elif flow_path is not None and name is not None: if name not in flow_path and (flow_path.endswith(".yaml") or flow_path.endswith(".yml")): - raise HTTPException(422, 'The name must be part of the flow path when a full path is provided') + raise HTTPException(422, "The name must be part of the flow path when a full path is provided") elif name in flow_path and not (flow_path.endswith(".yaml") or flow_path.endswith(".yml")): flow_path = str(Path(flow_path) / (name + ".yaml")) elif name not in flow_path and (name.endswith(".yaml") or name.endswith(".yml")): @@ -517,28 +518,28 @@ def close_flow(flow_id: int, current_user=Depends(get_current_active_user)) -> N flow_file_handler.delete_flow(flow_id, user_id=user_id) -@router.post('/update_settings/', tags=['transform']) -def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)): +@router.post("/update_settings/", tags=["transform"]) +def add_generic_settings(input_data: dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)): """A generic endpoint to update the settings of any node. This endpoint dynamically determines the correct Pydantic model and update function based on the `node_type` parameter. """ - input_data['user_id'] = current_user.id + input_data["user_id"] = current_user.id node_type = camel_case_to_snake_case(node_type) - flow_id = int(input_data.get('flow_id')) + flow_id = int(input_data.get("flow_id")) logger.info(f'Updating the data for flow: {flow_id}, node {input_data["node_id"]}') flow = flow_file_handler.get_flow(flow_id) if flow.flow_settings.is_running: - raise HTTPException(422, 'Flow is running') + raise HTTPException(422, "Flow is running") if flow is None: - raise HTTPException(404, 'could not find the flow') - add_func = getattr(flow, 'add_' + node_type) + raise HTTPException(404, "could not find the flow") + add_func = getattr(flow, "add_" + node_type) parsed_input = None - setting_name_ref = 'node' + node_type.replace('_', '') + setting_name_ref = "node" + node_type.replace("_", "") if add_func is None: - raise HTTPException(404, 'could not find the function') + raise HTTPException(404, "could not find the function") try: ref = get_node_model(setting_name_ref) if ref: @@ -546,73 +547,72 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use except Exception as e: raise HTTPException(421, str(e)) if parsed_input is None: - raise HTTPException(404, 'could not find the interface') + raise HTTPException(404, "could not find the interface") try: add_func(parsed_input) except Exception as e: logger.error(e) - raise HTTPException(419, str(f'error: {e}')) + raise HTTPException(419, str(f"error: {e}")) -@router.get('/files/available_flow_files', tags=['editor'], response_model=List[FileInfo]) +@router.get("/files/available_flow_files", tags=["editor"], response_model=list[FileInfo]) def get_list_of_saved_flows(path: str): """Scans a directory for saved flow files (`.flowfile`).""" try: # Validate path is within sandbox before proceeding - explorer = SecureFileExplorer( - start_path=storage.user_data_directory, - sandbox_root=storage.user_data_directory - ) + explorer = SecureFileExplorer(start_path=storage.user_data_directory, sandbox_root=storage.user_data_directory) validated_path = explorer.get_absolute_path(path) if validated_path is None: return [] - return get_files_from_directory(str(validated_path), types=['flowfile'], sandbox_root=storage.user_data_directory) + return get_files_from_directory( + str(validated_path), types=["flowfile"], sandbox_root=storage.user_data_directory + ) except: return [] -@router.get('/node_list', response_model=List[schemas.NodeTemplate]) -def get_node_list() -> List[schemas.NodeTemplate]: +@router.get("/node_list", response_model=list[schemas.NodeTemplate]) +def get_node_list() -> list[schemas.NodeTemplate]: """Retrieves the list of all available node types and their templates.""" return nodes_list -@router.get('/node', response_model=output_model.NodeData, tags=['editor']) +@router.get("/node", response_model=output_model.NodeData, tags=["editor"]) def get_node(flow_id: int, node_id: int, get_data: bool = False): """Retrieves the complete state and data preview for a single node.""" - logging.info(f'Getting node {node_id} from flow {flow_id}') + logging.info(f"Getting node {node_id} from flow {flow_id}") flow = flow_file_handler.get_flow(flow_id) node = flow.get_node(node_id) if node is None: - raise HTTPException(422, 'Not found') + raise HTTPException(422, "Not found") v = node.get_node_data(flow_id=flow.flow_id, include_example=get_data) return v -@router.post('/node/description/', tags=['editor']) +@router.post("/node/description/", tags=["editor"]) def update_description_node(flow_id: int, node_id: int, description: str = Body(...)): """Updates the description text for a specific node.""" try: node = flow_file_handler.get_flow(flow_id).get_node(node_id) except: - raise HTTPException(404, 'Could not find the node') + raise HTTPException(404, "Could not find the node") node.setting_input.description = description return True -@router.get('/node/description', tags=['editor']) +@router.get("/node/description", tags=["editor"]) def get_description_node(flow_id: int, node_id: int): """Retrieves the description text for a specific node.""" try: node = flow_file_handler.get_flow(flow_id).get_node(node_id) except: - raise HTTPException(404, 'Could not find the node') + raise HTTPException(404, "Could not find the node") if node is None: - raise HTTPException(404, 'Could not find the node') + raise HTTPException(404, "Could not find the node") return node.setting_input.description -@router.get('/node/data', response_model=output_model.TableExample, tags=['editor']) +@router.get("/node/data", response_model=output_model.TableExample, tags=["editor"]) def get_table_example(flow_id: int, node_id: int): """Retrieves a data preview (schema and sample rows) for a node's output.""" flow = flow_file_handler.get_flow(flow_id) @@ -620,8 +620,8 @@ def get_table_example(flow_id: int, node_id: int): return node.get_table_example(True) -@router.get('/node/downstream_node_ids', response_model=List[int], tags=['editor']) -async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]: +@router.get("/node/downstream_node_ids", response_model=list[int], tags=["editor"]) +async def get_downstream_node_ids(flow_id: int, node_id: int) -> list[int]: """Gets a list of all node IDs that are downstream dependencies of a given node.""" flow = flow_file_handler.get_flow(flow_id) node = flow.get_node(node_id) @@ -638,7 +638,7 @@ def import_saved_flow(flow_path: str, current_user=Depends(get_current_active_us return flow_file_handler.import_flow(Path(validated_path), user_id=user_id) -@router.get('/save_flow', tags=['editor']) +@router.get("/save_flow", tags=["editor"]) def save_flow(flow_id: int, flow_path: str = None): """Saves the current state of a flow to a `.yaml`.""" if flow_path is not None: @@ -647,55 +647,55 @@ def save_flow(flow_id: int, flow_path: str = None): flow.save_flow(flow_path=flow_path) -@router.get('/flow_data', tags=['manager']) -def get_flow_frontend_data(flow_id: Optional[int] = 1): +@router.get("/flow_data", tags=["manager"]) +def get_flow_frontend_data(flow_id: int | None = 1): """Retrieves the data needed to render the flow graph in the frontend.""" flow = flow_file_handler.get_flow(flow_id) if flow is None: - raise HTTPException(404, 'could not find the flow') + raise HTTPException(404, "could not find the flow") return flow.get_frontend_data() -@router.get('/flow_settings', tags=['manager'], response_model=schemas.FlowSettings) -def get_flow_settings(flow_id: Optional[int] = 1) -> schemas.FlowSettings: +@router.get("/flow_settings", tags=["manager"], response_model=schemas.FlowSettings) +def get_flow_settings(flow_id: int | None = 1) -> schemas.FlowSettings: """Retrieves the main settings for a flow.""" flow = flow_file_handler.get_flow(flow_id) if flow is None: - raise HTTPException(404, 'could not find the flow') + raise HTTPException(404, "could not find the flow") return flow.flow_settings -@router.post('/flow_settings', tags=['manager']) +@router.post("/flow_settings", tags=["manager"]) def update_flow_settings(flow_settings: schemas.FlowSettings): """Updates the main settings for a flow.""" flow = flow_file_handler.get_flow(flow_settings.flow_id) if flow is None: - raise HTTPException(404, 'could not find the flow') + raise HTTPException(404, "could not find the flow") flow.flow_settings = flow_settings -@router.get('/flow_data/v2', tags=['manager']) +@router.get("/flow_data/v2", tags=["manager"]) def get_vue_flow_data(flow_id: int) -> schemas.VueFlowInput: """Retrieves the flow data formatted for the Vue-based frontend.""" flow = flow_file_handler.get_flow(flow_id) if flow is None: - raise HTTPException(404, 'could not find the flow') + raise HTTPException(404, "could not find the flow") data = flow.get_vue_flow_input() return data -@router.get('/analysis_data/graphic_walker_input', tags=['analysis'], response_model=input_schema.NodeExploreData) +@router.get("/analysis_data/graphic_walker_input", tags=["analysis"], response_model=input_schema.NodeExploreData) def get_graphic_walker_input(flow_id: int, node_id: int): """Gets the data and configuration for the Graphic Walker data exploration tool.""" flow = flow_file_handler.get_flow(flow_id) node = flow.get_node(node_id) if node.results.analysis_data_generator is None: - logger.error('The data is not refreshed and available for analysis') - raise HTTPException(422, 'The data is not refreshed and available for analysis') + logger.error("The data is not refreshed and available for analysis") + raise HTTPException(422, "The data is not refreshed and available for analysis") return AnalyticsProcessor.process_graphic_walker_input(node) -@router.get('/custom_functions/instant_result', tags=[]) +@router.get("/custom_functions/instant_result", tags=[]) async def get_instant_function_result(flow_id: int, node_id: int, func_string: str): """Executes a simple, instant function on a node's data and returns the result.""" try: @@ -706,7 +706,7 @@ async def get_instant_function_result(flow_id: int, node_id: int, func_string: s raise HTTPException(status_code=500, detail=str(e)) -@router.get('/api/get_xlsx_sheet_names', tags=['excel_reader'], response_model=list[str]) +@router.get("/api/get_xlsx_sheet_names", tags=["excel_reader"], response_model=list[str]) async def get_excel_sheet_names(path: str) -> list[str] | None: """Retrieves the sheet names from an Excel file.""" validated_path = validate_path_under_cwd(path) @@ -714,13 +714,12 @@ async def get_excel_sheet_names(path: str) -> list[str] | None: if sheet_names: return sheet_names else: - raise HTTPException(404, 'File not found') + raise HTTPException(404, "File not found") @router.post("/validate_db_settings") async def validate_db_settings( - database_settings: input_schema.DatabaseSettings, - current_user=Depends(get_current_active_user) + database_settings: input_schema.DatabaseSettings, current_user=Depends(get_current_active_user) ): """Validates that a connection can be made to a database with the given settings.""" # Validate the query settings diff --git a/flowfile_core/flowfile_core/routes/user_defined_components.py b/flowfile_core/flowfile_core/routes/user_defined_components.py index 60f006feb..fab373bae 100644 --- a/flowfile_core/flowfile_core/routes/user_defined_components.py +++ b/flowfile_core/flowfile_core/routes/user_defined_components.py @@ -1,23 +1,24 @@ - import ast import re -from typing import Dict, Any, List, Optional from pathlib import Path +from typing import Any -from fastapi import APIRouter, HTTPException, Depends, UploadFile, File +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.responses import FileResponse from pydantic import BaseModel from flowfile_core import flow_file_handler + # Core modules from flowfile_core.auth.jwt import get_current_active_user from flowfile_core.configs import logger from flowfile_core.configs.node_store import ( CUSTOM_NODE_STORE, add_to_custom_node_store, - remove_from_custom_node_store, load_single_node_from_file, + remove_from_custom_node_store, ) + # File handling from flowfile_core.schemas import input_schema from flowfile_core.utils.utils import camel_case_to_snake_case @@ -31,6 +32,7 @@ class CustomNodeInfo(BaseModel): """Info about a custom node file.""" + file_name: str node_name: str = "" node_category: str = "" @@ -41,6 +43,7 @@ class CustomNodeInfo(BaseModel): class SaveCustomNodeRequest(BaseModel): """Request model for saving a custom node.""" + file_name: str code: str @@ -66,18 +69,18 @@ def get_simple_custom_object(flow_id: int, node_id: int): @router.post("/update_user_defined_node", tags=["transform"]) -def update_user_defined_node(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)): - input_data['user_id'] = current_user.id +def update_user_defined_node(input_data: dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)): + input_data["user_id"] = current_user.id node_type = camel_case_to_snake_case(node_type) - flow_id = int(input_data.get('flow_id')) + flow_id = int(input_data.get("flow_id")) logger.info(f'Updating the data for flow: {flow_id}, node {input_data["node_id"]}') flow = flow_file_handler.get_flow(flow_id) user_defined_model = CUSTOM_NODE_STORE.get(node_type) if not user_defined_model: raise HTTPException(status_code=404, detail=f"Node type '{node_type}' not found") - print('adding user defined node') + print("adding user defined node") print(input_data) - print('-----') + print("-----") user_defined_node_settings = input_schema.UserDefinedNode.model_validate(input_data) initialized_model = user_defined_model.from_settings(user_defined_node_settings.settings) @@ -97,22 +100,19 @@ def save_custom_node(request: SaveCustomNodeRequest): """ # Validate file name file_name = request.file_name - if not file_name.endswith('.py'): - file_name += '.py' + if not file_name.endswith(".py"): + file_name += ".py" # Sanitize file name - only allow alphanumeric, underscore, and .py extension - safe_name = re.sub(r'[^a-zA-Z0-9_]', '_', file_name[:-3]) + '.py' - if not safe_name or safe_name == '.py': + safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", file_name[:-3]) + ".py" + if not safe_name or safe_name == ".py": raise HTTPException(status_code=400, detail="Invalid file name") # Validate Python syntax try: ast.parse(request.code) except SyntaxError as e: - raise HTTPException( - status_code=400, - detail=f"Python syntax error at line {e.lineno}: {e.msg}" - ) + raise HTTPException(status_code=400, detail=f"Python syntax error at line {e.lineno}: {e.msg}") # Get the directory path nodes_dir = storage.user_defined_nodes_directory @@ -120,7 +120,7 @@ def save_custom_node(request: SaveCustomNodeRequest): # Write the file try: - with open(file_path, 'w', encoding='utf-8') as f: + with open(file_path, "w", encoding="utf-8") as f: f.write(request.code) logger.info(f"Saved custom node to {file_path}") except Exception as e: @@ -137,11 +137,7 @@ def save_custom_node(request: SaveCustomNodeRequest): logger.warning(f"Node saved but failed to load: {e}") # Don't fail the request - the file is saved, it just couldn't be loaded yet - return { - "success": True, - "file_name": safe_name, - "message": f"Node saved successfully to {safe_name}" - } + return {"success": True, "file_name": safe_name, "message": f"Node saved successfully to {safe_name}"} def _extract_node_info_from_file(file_path: Path) -> CustomNodeInfo: @@ -149,7 +145,7 @@ def _extract_node_info_from_file(file_path: Path) -> CustomNodeInfo: info = CustomNodeInfo(file_name=file_path.name) try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding="utf-8") as f: content = f.read() tree = ast.parse(content) @@ -199,14 +195,14 @@ def _extract_node_info_from_file(file_path: Path) -> CustomNodeInfo: return info -@router.get("/list-custom-nodes", summary="List all custom nodes", response_model=List[CustomNodeInfo]) -def list_custom_nodes() -> List[CustomNodeInfo]: +@router.get("/list-custom-nodes", summary="List all custom nodes", response_model=list[CustomNodeInfo]) +def list_custom_nodes() -> list[CustomNodeInfo]: """ List all custom node Python files in the user-defined nodes directory. Returns basic metadata extracted from each file. """ nodes_dir = storage.user_defined_nodes_directory - nodes: List[CustomNodeInfo] = [] + nodes: list[CustomNodeInfo] = [] if not nodes_dir.exists(): return nodes @@ -223,35 +219,29 @@ def list_custom_nodes() -> List[CustomNodeInfo]: @router.get("/get-custom-node/{file_name}", summary="Get custom node details") -def get_custom_node(file_name: str) -> Dict[str, Any]: +def get_custom_node(file_name: str) -> dict[str, Any]: """ Get the full content and parsed metadata of a custom node file. This endpoint is used by the Node Designer to load an existing node for editing. """ # Sanitize file name - if not file_name.endswith('.py'): - file_name += '.py' + if not file_name.endswith(".py"): + file_name += ".py" - safe_name = re.sub(r'[^a-zA-Z0-9_.]', '_', file_name) + safe_name = re.sub(r"[^a-zA-Z0-9_.]", "_", file_name) file_path = storage.user_defined_nodes_directory / safe_name if not file_path.exists(): raise HTTPException(status_code=404, detail=f"Node file '{safe_name}' not found") try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding="utf-8") as f: content = f.read() except Exception as e: raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}") # Parse the file to extract metadata and sections - result = { - "file_name": safe_name, - "content": content, - "metadata": {}, - "sections": [], - "processCode": "" - } + result = {"file_name": safe_name, "content": content, "metadata": {}, "sections": [], "processCode": ""} try: tree = ast.parse(content) @@ -304,10 +294,10 @@ def get_custom_node(file_name: str) -> Dict[str, Any]: if isinstance(item, ast.FunctionDef) and item.name == "process": # Get the source code of the process method start_line = item.lineno - 1 - end_line = item.end_lineno if hasattr(item, 'end_lineno') else start_line + 20 - lines = content.split('\n') + end_line = item.end_lineno if hasattr(item, "end_lineno") else start_line + 20 + lines = content.split("\n") process_lines = lines[start_line:end_line] - result["processCode"] = '\n'.join(process_lines) + result["processCode"] = "\n".join(process_lines) break break @@ -320,16 +310,16 @@ def get_custom_node(file_name: str) -> Dict[str, Any]: @router.delete("/delete-custom-node/{file_name}", summary="Delete a custom node") -def delete_custom_node(file_name: str) -> Dict[str, Any]: +def delete_custom_node(file_name: str) -> dict[str, Any]: """ Delete a custom node Python file from the user-defined nodes directory. This also attempts to unregister the node from the node store. """ # Sanitize file name - if not file_name.endswith('.py'): - file_name += '.py' + if not file_name.endswith(".py"): + file_name += ".py" - safe_name = re.sub(r'[^a-zA-Z0-9_.]', '_', file_name) + safe_name = re.sub(r"[^a-zA-Z0-9_.]", "_", file_name) file_path = storage.user_defined_nodes_directory / safe_name if not file_path.exists(): @@ -339,12 +329,14 @@ def delete_custom_node(file_name: str) -> Dict[str, Any]: try: info = _extract_node_info_from_file(file_path) file_stem = file_path.stem # filename without .py extension - logger.info(f"Extracted node info: node_name='{info.node_name}', file_name='{info.file_name}', file_stem='{file_stem}'") + logger.info( + f"Extracted node info: node_name='{info.node_name}', file_name='{info.file_name}', file_stem='{file_stem}'" + ) # Use the centralized remove function which cleans up all stores # Pass both the computed key from node_name and the file_stem as fallback if info.node_name: - node_type_key = info.node_name.lower().replace(' ', '_') + node_type_key = info.node_name.lower().replace(" ", "_") logger.info(f"Computed node_type_key: '{node_type_key}'") else: node_type_key = file_stem @@ -365,33 +357,31 @@ def delete_custom_node(file_name: str) -> Dict[str, Any]: logger.error(f"Failed to delete custom node file: {e}") raise HTTPException(status_code=500, detail=f"Failed to delete file: {str(e)}") - return { - "success": True, - "file_name": safe_name, - "message": f"Node '{safe_name}' deleted successfully" - } + return {"success": True, "file_name": safe_name, "message": f"Node '{safe_name}' deleted successfully"} # ==================== Custom Icon Endpoints ==================== + class IconInfo(BaseModel): """Info about a custom icon file.""" + file_name: str is_custom: bool = True -ALLOWED_ICON_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.svg', '.gif', '.webp'} +ALLOWED_ICON_EXTENSIONS = {".png", ".jpg", ".jpeg", ".svg", ".gif", ".webp"} MAX_ICON_SIZE = 5 * 1024 * 1024 # 5MB -@router.get("/list-icons", summary="List all available icons", response_model=List[IconInfo]) -def list_icons() -> List[IconInfo]: +@router.get("/list-icons", summary="List all available icons", response_model=list[IconInfo]) +def list_icons() -> list[IconInfo]: """ List all icon files available for custom nodes. Returns icons from the user_defined_nodes/icons directory. """ icons_dir = storage.user_defined_nodes_icons - icons: List[IconInfo] = [] + icons: list[IconInfo] = [] if not icons_dir.exists(): return icons @@ -406,7 +396,7 @@ def list_icons() -> List[IconInfo]: @router.post("/upload-icon", summary="Upload a custom icon") -async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]: +async def upload_icon(file: UploadFile = File(...)) -> dict[str, Any]: """ Upload a new icon file to the user_defined_nodes/icons directory. @@ -419,8 +409,7 @@ async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]: file_ext = Path(file.filename).suffix.lower() if file_ext not in ALLOWED_ICON_EXTENSIONS: raise HTTPException( - status_code=400, - detail=f"Invalid file type. Allowed types: {', '.join(ALLOWED_ICON_EXTENSIONS)}" + status_code=400, detail=f"Invalid file type. Allowed types: {', '.join(ALLOWED_ICON_EXTENSIONS)}" ) # Read file content @@ -429,12 +418,11 @@ async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]: # Validate file size if len(content) > MAX_ICON_SIZE: raise HTTPException( - status_code=400, - detail=f"File too large. Maximum size is {MAX_ICON_SIZE // (1024 * 1024)}MB" + status_code=400, detail=f"File too large. Maximum size is {MAX_ICON_SIZE // (1024 * 1024)}MB" ) # Sanitize filename - preserve hyphens, dots, and underscores - safe_name = re.sub(r'[^a-zA-Z0-9_.\-]', '_', file.filename) + safe_name = re.sub(r"[^a-zA-Z0-9_.\-]", "_", file.filename) if not safe_name: raise HTTPException(status_code=400, detail="Invalid file name") @@ -447,7 +435,7 @@ async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]: # Write the file try: - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: f.write(content) logger.info(f"Uploaded icon: {file_path} (size: {len(content)} bytes)") except Exception as e: @@ -458,7 +446,7 @@ async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]: "success": True, "file_name": safe_name, "path": str(file_path), - "message": f"Icon '{safe_name}' uploaded successfully" + "message": f"Icon '{safe_name}' uploaded successfully", } @@ -469,7 +457,7 @@ def get_icon(file_name: str) -> FileResponse: Returns the icon file for display in the UI. """ # Sanitize file name - preserve hyphens, dots, and underscores - safe_name = re.sub(r'[^a-zA-Z0-9_.\-]', '_', file_name) + safe_name = re.sub(r"[^a-zA-Z0-9_.\-]", "_", file_name) icons_dir = storage.user_defined_nodes_icons file_path = icons_dir / safe_name @@ -490,29 +478,25 @@ def get_icon(file_name: str) -> FileResponse: # Determine content type content_type_map = { - '.png': 'image/png', - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.svg': 'image/svg+xml', - '.gif': 'image/gif', - '.webp': 'image/webp', + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".svg": "image/svg+xml", + ".gif": "image/gif", + ".webp": "image/webp", } - content_type = content_type_map.get(file_path.suffix.lower(), 'application/octet-stream') + content_type = content_type_map.get(file_path.suffix.lower(), "application/octet-stream") - return FileResponse( - path=file_path, - media_type=content_type, - filename=safe_name - ) + return FileResponse(path=file_path, media_type=content_type, filename=safe_name) @router.delete("/delete-icon/{file_name}", summary="Delete a custom icon") -def delete_icon(file_name: str) -> Dict[str, Any]: +def delete_icon(file_name: str) -> dict[str, Any]: """ Delete a custom icon file from the icons directory. """ # Sanitize file name - preserve hyphens, dots, and underscores - safe_name = re.sub(r'[^a-zA-Z0-9_.\-]', '_', file_name) + safe_name = re.sub(r"[^a-zA-Z0-9_.\-]", "_", file_name) icons_dir = storage.user_defined_nodes_icons file_path = icons_dir / safe_name @@ -527,8 +511,4 @@ def delete_icon(file_name: str) -> Dict[str, Any]: logger.error(f"Failed to delete icon: {e}") raise HTTPException(status_code=500, detail=f"Failed to delete icon: {str(e)}") - return { - "success": True, - "file_name": safe_name, - "message": f"Icon '{safe_name}' deleted successfully" - } + return {"success": True, "file_name": safe_name, "message": f"Icon '{safe_name}' deleted successfully"} diff --git a/flowfile_core/flowfile_core/schemas/transform_schema.py b/flowfile_core/flowfile_core/schemas/transform_schema.py index 2d5d87a43..0720fe988 100644 --- a/flowfile_core/flowfile_core/schemas/transform_schema.py +++ b/flowfile_core/flowfile_core/schemas/transform_schema.py @@ -519,7 +519,7 @@ def _parse_join_mapping(join_mapping: Any) -> list[JoinMap]: result.append(jm) elif isinstance(jm, dict): result.append(JoinMap(**jm)) - elif isinstance(jm, (tuple, list)) and len(jm) == 2: + elif isinstance(jm, tuple | list) and len(jm) == 2: result.append(JoinMap(left_col=jm[0], right_col=jm[1])) elif isinstance(jm, str): result.append(JoinMap(left_col=jm, right_col=jm)) @@ -632,7 +632,7 @@ def _parse_join_mapping(join_mapping: Any) -> list[JoinMap]: result.append(jm) elif isinstance(jm, dict): result.append(JoinMap(**jm)) - elif isinstance(jm, (tuple, list)) and len(jm) == 2: + elif isinstance(jm, tuple | list) and len(jm) == 2: result.append(JoinMap(left_col=jm[0], right_col=jm[1])) elif isinstance(jm, str): result.append(JoinMap(left_col=jm, right_col=jm)) @@ -1522,7 +1522,7 @@ def parse_fuzz_mapping( fuzz_mapping: list[FuzzyMapping] | tuple[str, str] | str | FuzzyMapping | list[dict], ) -> list[FuzzyMapping]: """Parses various input formats into a list of FuzzyMapping objects.""" - if isinstance(fuzz_mapping, (tuple, list)): + if isinstance(fuzz_mapping, tuple | list): if len(fuzz_mapping) == 0: raise ValueError("Fuzzy mapping cannot be empty") diff --git a/flowfile_core/tests/flowfile/flowfile_table/fuzzy_mathcing/test_prepare_for_fuzzy_match.py b/flowfile_core/tests/flowfile/flowfile_table/fuzzy_mathcing/test_prepare_for_fuzzy_match.py index d9a477f4e..859e178cf 100644 --- a/flowfile_core/tests/flowfile/flowfile_table/fuzzy_mathcing/test_prepare_for_fuzzy_match.py +++ b/flowfile_core/tests/flowfile/flowfile_table/fuzzy_mathcing/test_prepare_for_fuzzy_match.py @@ -22,4 +22,3 @@ def test_prepare_for_fuzzy_match(): assert fuzzy_match_input_manager.used_join_mapping[0].left_col == '_FLOWFILE_JOIN_KEY_LEFT_name', 'Left column should still be named name' assert fuzzy_match_input_manager.used_join_mapping[0].right_col == '_FLOWFILE_JOIN_KEY_RIGHT_name_right', 'Right column should be renamed to name_right' - diff --git a/flowfile_core/tests/flowfile/test_flowfile.py b/flowfile_core/tests/flowfile/test_flowfile.py index b506e80a9..19484f665 100644 --- a/flowfile_core/tests/flowfile/test_flowfile.py +++ b/flowfile_core/tests/flowfile/test_flowfile.py @@ -1713,4 +1713,3 @@ def test_fetch_before_run_debug(): example_data_after_run = node.get_table_example(True).data assert len(example_data_after_run) > 0, "There should be data after fetch operation" - diff --git a/flowfile_frame/flow_frame_stub_generator.py b/flowfile_frame/flow_frame_stub_generator.py index 47f07f83b..2db7da2d9 100644 --- a/flowfile_frame/flow_frame_stub_generator.py +++ b/flowfile_frame/flow_frame_stub_generator.py @@ -52,7 +52,7 @@ def format_default_value(param: inspect.Parameter) -> str | None: default = param.default - if isinstance(default, (str, int, float, bool, type(None))): + if isinstance(default, str | int | float | bool | type(None)): return repr(default) type_name = type(default).__name__ diff --git a/flowfile_frame/flowfile_frame/adding_expr.py b/flowfile_frame/flowfile_frame/adding_expr.py index 6e5768dfe..152d02793 100644 --- a/flowfile_frame/flowfile_frame/adding_expr.py +++ b/flowfile_frame/flowfile_frame/adding_expr.py @@ -89,16 +89,15 @@ def wrapper(self: Expr, *args, **kwargs): kwargs_repr = ", ".join(kwargs_representations) if args_repr and kwargs_repr: - params_repr = f"{args_repr}, {kwargs_repr}" + pass elif args_repr: - params_repr = args_repr + pass elif kwargs_repr: - params_repr = kwargs_repr + pass else: - params_repr = "" + pass # Create the repr string for this method call - new_repr = f"{self._repr_str}.{method_name}({params_repr})" # Methods that typically change the aggregation status or complexity agg_methods = { @@ -227,7 +226,7 @@ def passthrough_method(self, *args, **kwargs): convertable_to_code = True # Process positional arguments - for i, arg in enumerate(args): + for _i, arg in enumerate(args): if callable(arg) and not isinstance(arg, type): # Try to get function source try: @@ -280,15 +279,14 @@ def passthrough_method(self, *args, **kwargs): kwargs_repr = ", ".join(kwargs_representations) if args_repr and kwargs_repr: - params_repr = f"{args_repr}, {kwargs_repr}" + pass elif args_repr: - params_repr = args_repr + pass elif kwargs_repr: - params_repr = kwargs_repr + pass else: - params_repr = "" + pass # Create a representation string - new_repr = f"{self._repr_str}.{method_name}({params_repr})" # self._repr_str = new_repr # Return a new expression with the convertable_to_code flag set appropriately result = self._create_next_expr( diff --git a/flowfile_frame/flowfile_frame/expr.py b/flowfile_frame/flowfile_frame/expr.py index 193a74aad..827c312a8 100644 --- a/flowfile_frame/flowfile_frame/expr.py +++ b/flowfile_frame/flowfile_frame/expr.py @@ -531,7 +531,7 @@ def _create_binary_op_expr(self, op_symbol: str, other: Any, result_expr: pl.Exp other_expr, other_repr = _get_expr_and_repr(other) - if other_expr is None and not isinstance(other, (int, float, str, bool, type(None))): + if other_expr is None and not isinstance(other, int | float | str | bool | type(None)): raise ValueError( f"Cannot perform binary operation '{op_symbol}' with operand without underlying polars expression or literal value: {other_repr}" ) @@ -758,7 +758,7 @@ def __rmod__(self, other): def __rpow__(self, other): other_expr, other_repr = _get_expr_and_repr(other) new_repr = f"({other_repr} ** {self._repr_str})" - base_expr = pl.lit(other) if not isinstance(other, (Expr, pl.Expr)) else other_expr + base_expr = pl.lit(other) if not isinstance(other, Expr | pl.Expr) else other_expr res_expr = base_expr.pow(self.expr) if self.expr is not None and base_expr is not None else None return Expr(res_expr, None, repr_str=new_repr, agg_func=None, is_complex=True) @@ -933,7 +933,7 @@ def fill_nan(self, value): @staticmethod def _get_expr_repr(expr): """Helper to get appropriate string representation for an expression""" - if isinstance(expr, (Expr, Column)): + if isinstance(expr, Expr | Column): return expr._repr_str elif isinstance(expr, str): return f"pl.col('{expr}')" diff --git a/flowfile_frame/flowfile_frame/flow_frame.py b/flowfile_frame/flowfile_frame/flow_frame.py index 9b3cb09df..78876abc8 100644 --- a/flowfile_frame/flowfile_frame/flow_frame.py +++ b/flowfile_frame/flowfile_frame/flow_frame.py @@ -4,7 +4,7 @@ import os import re from collections.abc import Iterable, Iterator, Mapping -from typing import Any, Literal, Optional, Union, get_args, get_origin +from typing import Any, Literal, Union, get_args, get_origin import polars as pl from pl_fuzzy_frame_match import FuzzyMapping @@ -198,7 +198,7 @@ def create_from_any_type( ) pl_data = pl_df.lazy() except Exception as e: - raise ValueError(f"Could not dconvert data to a polars DataFrame: {e}") + raise ValueError(f"Could not dconvert data to a polars DataFrame: {e}") from e # Create a FlowDataEngine to get data in the right format for manual input flow_table = FlowDataEngine(raw_data=pl_data) raw_data_format = input_schema.RawData( @@ -239,7 +239,7 @@ def __new__( node_id: int | None = None, parent_node_id: int | None = None, **kwargs, # Accept and ignore any other kwargs for API compatibility - ) -> "FlowFrame": + ) -> FlowFrame: """ Unified constructor for FlowFrame. @@ -291,7 +291,7 @@ def __new__( ) pl_data = pl_df.lazy() except Exception as e: - raise ValueError(f"Could not convert data to a Polars DataFrame: {e}") + raise ValueError(f"Could not convert data to a Polars DataFrame: {e}") from e flow_table = FlowDataEngine(raw_data=pl_data) raw_data_format = input_schema.RawData( @@ -344,8 +344,8 @@ def _create_child_frame(self, new_node_id): node_id=new_node_id, parent_node_id=self.node_id, ) - except AttributeError: - raise ValueError("Could not execute the function") + except AttributeError as e: + raise ValueError("Could not execute the function") from e @staticmethod def _generate_sort_polars_code( @@ -384,7 +384,7 @@ def sort( multithreaded: bool = True, maintain_order: bool = False, description: str | None = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Sort the dataframe by the given columns. """ @@ -405,7 +405,7 @@ def sort( if maintain_order or not multithreaded: use_polars_code_path = True - is_nulls_last_list = isinstance(nulls_last, (list, tuple)) + is_nulls_last_list = isinstance(nulls_last, list | tuple) if is_nulls_last_list and any(val for val in nulls_last if val is not False): use_polars_code_path = True elif not is_nulls_last_list and nulls_last is not False: @@ -550,7 +550,8 @@ def _add_polars_code( target_obj = getattr(self.data, effective_method_name)(*group_expr_list, **group_kwargs) if not pl_expr_list: raise ValueError( - "Aggregation expressions (polars_expr) are required for group_by().agg() in serialization fallback." + "Aggregation expressions (polars_expr) are required for " + "group_by().agg() in serialization fallback." ) result_lazyframe_or_expr = target_obj.agg(*pl_expr_list, **current_kwargs_expr) elif effective_method_name: @@ -559,7 +560,8 @@ def _add_polars_code( ) else: raise ValueError( - "Cannot execute Polars operation: method_name is missing and could not be inferred for serialization fallback." + "Cannot execute Polars operation: method_name is missing and could not be " + "inferred for serialization fallback." ) try: if isinstance(result_lazyframe_or_expr, pl.LazyFrame): @@ -615,7 +617,7 @@ def join( coalesce: bool = None, maintain_order: Literal[None, "left", "right", "left_right", "right_left"] = None, description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Add a join operation to the Logical Plan. @@ -714,7 +716,7 @@ def _should_use_polars_code_for_join(self, maintain_order, coalesce, nulls_equal and suffix == "_right" ) - def _ensure_same_graph(self, other: "FlowFrame") -> None: + def _ensure_same_graph(self, other: FlowFrame) -> None: """Ensure both FlowFrames are in the same graph, combining if necessary.""" if self.flow_graph.flow_id != other.flow_graph.flow_id: combined_graph, node_mappings = combine_flow_graphs_with_mapping(self.flow_graph, other.flow_graph) @@ -754,7 +756,7 @@ def _parse_join_columns( def _execute_polars_code_join( self, - other: "FlowFrame", + other: FlowFrame, new_node_id: int, on: list[str | Column] | str | Column, left_on: list[str | Column] | str | Column, @@ -768,7 +770,7 @@ def _execute_polars_code_join( coalesce: bool, maintain_order: Literal[None, "left", "right", "left_right", "right_left"], description: str, - ) -> "FlowFrame": + ) -> FlowFrame: """Execute join using Polars code approach.""" # Build the code arguments code_kwargs = self._build_polars_join_kwargs( @@ -843,12 +845,12 @@ def format_column_list(cols): def _execute_native_join( self, - other: "FlowFrame", + other: FlowFrame, new_node_id: int, join_mappings: list | None, how: str, description: str, - ) -> "FlowFrame": + ) -> FlowFrame: """Execute join using native FlowFile join nodes.""" # Create select inputs for both frames @@ -896,9 +898,9 @@ def _execute_native_join( def _add_cross_join_node( self, new_node_id: int, - join_input: "transform_schema.CrossJoinInput", + join_input: transform_schema.CrossJoinInput, description: str, - other: "FlowFrame", + other: FlowFrame, ) -> None: """Add a cross join node to the graph.""" cross_join_settings = input_schema.NodeCrossJoin( @@ -916,9 +918,9 @@ def _add_cross_join_node( def _add_regular_join_node( self, new_node_id: int, - join_input: "transform_schema.JoinInput", + join_input: transform_schema.JoinInput, description: str, - other: "FlowFrame", + other: FlowFrame, ) -> None: """Add a regular join node to the graph.""" join_settings = input_schema.NodeJoin( @@ -935,7 +937,7 @@ def _add_regular_join_node( ) self.flow_graph.add_join(join_settings) - def _add_number_of_records(self, new_node_id: int, description: str = None) -> "FlowFrame": + def _add_number_of_records(self, new_node_id: int, description: str = None) -> FlowFrame: node_number_of_records = input_schema.NodeRecordCount( flow_id=self.flow_graph.flow_id, node_id=new_node_id, @@ -948,7 +950,7 @@ def _add_number_of_records(self, new_node_id: int, description: str = None) -> " self.flow_graph.add_record_count(node_number_of_records) return self._create_child_frame(new_node_id) - def rename(self, mapping: Mapping[str, str], *, strict: bool = True, description: str = None) -> "FlowFrame": + def rename(self, mapping: Mapping[str, str], *, strict: bool = True, description: str = None) -> FlowFrame: """Rename columns based on a mapping or function.""" return self.select( [col(old_name).alias(new_name) for old_name, new_name in mapping.items()], @@ -958,7 +960,7 @@ def rename(self, mapping: Mapping[str, str], *, strict: bool = True, description def select( self, *columns: str | Expr | Selector, description: str | None = None, _keep_missing: bool = False - ) -> "FlowFrame": + ) -> FlowFrame: """ Select columns from the frame. """ @@ -1068,7 +1070,7 @@ def filter( flowfile_formula: str | None = None, description: str | None = None, **constraints: Any, - ) -> "FlowFrame": + ) -> FlowFrame: """ Filter rows based on a predicate. """ @@ -1083,7 +1085,7 @@ def filter( processed_predicates = [] for pred_item in predicates: - if isinstance(pred_item, (tuple, list, Iterator)): + if isinstance(pred_item, tuple | list | Iterator): # If it's a sequence, extend the processed_predicates with its elements processed_predicates.extend(list(pred_item)) else: @@ -1184,7 +1186,7 @@ def write_parquet( description: str = None, convert_to_absolute_path: bool = True, **kwargs: Any, - ) -> "FlowFrame": + ) -> FlowFrame: """ Write the data to a Parquet file. Creates a standard Output node if only 'path' and standard options are provided. Falls back to a Polars Code node @@ -1207,7 +1209,7 @@ def write_parquet( """ new_node_id = generate_node_id() - is_path_input = isinstance(path, (str, os.PathLike)) + is_path_input = isinstance(path, str | os.PathLike) if isinstance(path, os.PathLike): file_str = str(path) elif isinstance(path, str): @@ -1275,9 +1277,9 @@ def write_csv( description: str = None, convert_to_absolute_path: bool = True, **kwargs: Any, - ) -> "FlowFrame": + ) -> FlowFrame: new_node_id = generate_node_id() - is_path_input = isinstance(file, (str, os.PathLike)) + is_path_input = isinstance(file, str | os.PathLike) if isinstance(file, os.PathLike): file_str = str(file) elif isinstance(file, str): @@ -1346,7 +1348,7 @@ def write_parquet_to_cloud_storage( connection_name: str | None = None, compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy", description: str | None = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Write the data frame to cloud storage in Parquet format. @@ -1380,7 +1382,7 @@ def write_csv_to_cloud_storage( delimiter: str = ";", encoding: CsvEncoding = "utf8", description: str | None = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Write the data frame to cloud storage in CSV format. @@ -1415,7 +1417,7 @@ def write_delta( connection_name: str | None = None, write_mode: Literal["overwrite", "append"] = "overwrite", description: str | None = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Write the data frame to cloud storage in Delta Lake format. @@ -1445,7 +1447,7 @@ def write_json_to_cloud_storage( path: str, connection_name: str | None = None, description: str | None = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Write the data frame to cloud storage in JSON format. @@ -1490,7 +1492,7 @@ def group_by(self, *by, description: str = None, maintain_order=False, **named_b by_cols.append(col_expr) elif isinstance(col_expr, Selector): by_cols.append(col_expr) - elif isinstance(col_expr, (list, tuple)): + elif isinstance(col_expr, list | tuple): by_cols.extend(col_expr) for new_name, col_expr in named_by.items(): @@ -1523,7 +1525,7 @@ def collect(self, *args, **kwargs) -> pl.DataFrame: return self.data.collect(*args, **kwargs) return self.data - def _with_flowfile_formula(self, flowfile_formula: str, output_column_name, description: str = None) -> "FlowFrame": + def _with_flowfile_formula(self, flowfile_formula: str, output_column_name, description: str = None) -> FlowFrame: new_node_id = generate_node_id() function_settings = input_schema.NodeFormula( flow_id=self.flow_graph.flow_id, @@ -1552,7 +1554,7 @@ def head(self, n: int, description: str = None): def limit(self, n: int, description: str = None): return self.head(n, description) - def cache(self) -> "FlowFrame": + def cache(self) -> FlowFrame: setting_input = self.get_node_settings().setting_input setting_input.cache_results = True self.data.cache() @@ -1572,7 +1574,7 @@ def pivot( sort_columns: bool = False, separator: str = "_", description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Pivot a DataFrame from long to wide format. @@ -1663,7 +1665,7 @@ def pivot( code = f""" # Perform pivot operation result = input_df.pivot( - on={on_repr}, + on={on_repr}, index={index_repr}, values={values_repr}, aggregate_function='{aggregate_function}', @@ -1694,7 +1696,7 @@ def unpivot( variable_name: str = "variable", value_name: str = "value", description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Unpivot a DataFrame from wide to long format. @@ -1729,7 +1731,7 @@ def unpivot( can_use_native = True if on is None: value_columns = [] - elif isinstance(on, (str, Selector)): + elif isinstance(on, str | Selector): if isinstance(on, Selector): can_use_native = False value_columns = [on] @@ -1775,7 +1777,7 @@ def unpivot( code = f""" # Perform unpivot operation output_df = input_df.unpivot( - on={on_repr}, + on={on_repr}, index={index_repr}, variable_name="{variable_name}", value_name="{value_name}" @@ -1795,12 +1797,12 @@ def unpivot( def concat( self, - other: "FlowFrame" | list["FlowFrame"], + other: FlowFrame | list[FlowFrame], how: str = "vertical", rechunk: bool = False, parallel: bool = True, description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Combine multiple FlowFrames into a single FlowFrame. @@ -1913,7 +1915,7 @@ def concat( def _detect_cum_count_record_id( self, expr: Any, new_node_id: int, description: str | None = None - ) -> tuple[bool, Optional["FlowFrame"]]: + ) -> tuple[bool, FlowFrame | None]: """ Detect if the expression is a cum_count operation and use record_id if possible. @@ -2032,7 +2034,7 @@ def with_columns( output_column_names: list[str] | None = None, description: str | None = None, **named_exprs: Expr | Any, # Allow Any for implicit lit conversion - ) -> "FlowFrame": + ) -> FlowFrame: """ Add or replace columns in the DataFrame. """ @@ -2110,7 +2112,7 @@ def with_columns( else: raise ValueError("Either exprs/named_exprs or flowfile_formulas with output_column_names must be provided") - def with_row_index(self, name: str = "index", offset: int = 0, description: str = None) -> "FlowFrame": + def with_row_index(self, name: str = "index", offset: int = 0, description: str = None) -> FlowFrame: """ Add a row index as the first column in the DataFrame. @@ -2166,7 +2168,7 @@ def explode( columns: str | Column | Iterable[str | Column], *more_columns: str | Column, description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Explode the dataframe to long format by exploding the given columns. @@ -2190,7 +2192,7 @@ def explode( all_columns = [] - if isinstance(columns, (list, tuple)): + if isinstance(columns, list | tuple): all_columns.extend([col.column_name if isinstance(col, Column) else col for col in columns]) else: all_columns.append(columns.column_name if isinstance(columns, Column) else columns) @@ -2219,10 +2221,10 @@ def explode( def fuzzy_match( self, - other: "FlowFrame", + other: FlowFrame, fuzzy_mappings: list[FuzzyMapping], description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: self._ensure_same_graph(other) # Step 3: Generate new node ID @@ -2253,7 +2255,7 @@ def text_to_rows( delimiter: str = None, split_by_column: str = None, description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Split text in a column into multiple rows. @@ -2314,12 +2316,12 @@ def text_to_rows( def unique( self, - subset: Union[str, "Expr", list[Union[str, "Expr"]]] = None, + subset: str | Expr | list[str | Expr] = None, *, keep: Literal["first", "last", "any", "none"] = "any", maintain_order: bool = False, description: str = None, - ) -> "FlowFrame": + ) -> FlowFrame: """ Drop duplicate rows from this dataframe. @@ -2353,7 +2355,7 @@ def unique( can_use_native = True if subset is not None: # Convert to list if single item - if not isinstance(subset, (list, tuple)): + if not isinstance(subset, list | tuple): subset = [subset] # Extract column names @@ -2395,7 +2397,7 @@ def unique( # Generate polars code for more complex cases if subset is None: subset_str = "None" - elif isinstance(subset, (list, tuple)): + elif isinstance(subset, list | tuple): # Format each item in the subset list items = [] for item in subset: diff --git a/flowfile_frame/flowfile_frame/flow_frame_methods.py b/flowfile_frame/flowfile_frame/flow_frame_methods.py index 33eb16975..59f36d2e1 100644 --- a/flowfile_frame/flowfile_frame/flow_frame_methods.py +++ b/flowfile_frame/flowfile_frame/flow_frame_methods.py @@ -144,15 +144,15 @@ def read_csv( flow_graph = create_flow_graph() flow_id = flow_graph.flow_id current_source_path_for_native = None - if isinstance(source, (str, os.PathLike)): + if isinstance(source, str | os.PathLike): current_source_path_for_native = str(source) if "~" in current_source_path_for_native: current_source_path_for_native = os.path.expanduser(current_source_path_for_native) - elif isinstance(source, list) and all(isinstance(s, (str, os.PathLike)) for s in source): + elif isinstance(source, list) and all(isinstance(s, str | os.PathLike) for s in source): current_source_path_for_native = str(source[0]) if source else None if current_source_path_for_native and "~" in current_source_path_for_native: current_source_path_for_native = os.path.expanduser(current_source_path_for_native) - elif isinstance(source, (io.BytesIO, io.StringIO)): + elif isinstance(source, io.BytesIO | io.StringIO): logger.warning("Read from bytes io from csv not supported, converting data to raw data") return from_dict(pl.read_csv(source), flow_graph=flow_graph, description=description) actual_infer_schema_length: int | None @@ -259,9 +259,9 @@ def read_csv( **other_options, ) polars_code_node_description = description or "Read CSV with Polars scan_csv" - if isinstance(source, (str, os.PathLike)): + if isinstance(source, str | os.PathLike): polars_code_node_description = description or f"Read CSV with Polars scan_csv from {Path(source).name}" - elif isinstance(source, list) and source and isinstance(source[0], (str, os.PathLike)): + elif isinstance(source, list) and source and isinstance(source[0], str | os.PathLike): polars_code_node_description = ( description or f"Read CSV with Polars scan_csv from {Path(source[0]).name} (and possibly others)" ) @@ -316,7 +316,7 @@ def _build_polars_code_args( **other_options: Any, ) -> str: source_repr: str - if isinstance(source, (str, Path)): + if isinstance(source, str | Path): source_repr = repr(str(source)) elif isinstance(source, list): source_repr = repr([str(p) for p in source]) @@ -362,7 +362,7 @@ def _build_polars_code_args( all_vars = locals() kwargs_list = [] - for param_name_key, (default_value, format_func) in param_mapping.items(): + for param_name_key, (_default_value, format_func) in param_mapping.items(): value = all_vars.get(param_name_key) formatted_value = format_func(value) kwargs_list.append(f"{param_name_key}={formatted_value}") @@ -398,7 +398,7 @@ def read_parquet( A FlowFrame with the Parquet data """ if "~" in source: - file_path = os.path.expanduser(source) + os.path.expanduser(source) node_id = generate_node_id() if flow_graph is None: diff --git a/flowfile_frame/flowfile_frame/group_frame.py b/flowfile_frame/flowfile_frame/group_frame.py index 80059bbe0..121e58f20 100644 --- a/flowfile_frame/flowfile_frame/group_frame.py +++ b/flowfile_frame/flowfile_frame/group_frame.py @@ -41,7 +41,7 @@ def _create_expr_col(col_: Any) -> Expr: """Convert various column specifications to Expr objects.""" if isinstance(col_, str): return col(col_) - elif isinstance(col_, (Column, Expr)): + elif isinstance(col_, Column | Expr): return col_ else: return lit(col_) diff --git a/flowfile_frame/flowfile_frame/join.py b/flowfile_frame/flowfile_frame/join.py index 87450a7a0..c3e5b385d 100644 --- a/flowfile_frame/flowfile_frame/join.py +++ b/flowfile_frame/flowfile_frame/join.py @@ -16,7 +16,7 @@ def _normalize_columns_to_list(columns): return [] elif isinstance(columns, str): return [columns] - elif isinstance(columns, (list, tuple)): + elif isinstance(columns, list | tuple): return list(columns) else: return [columns] # Single non-string item diff --git a/flowfile_frame/flowfile_frame/lazy.py b/flowfile_frame/flowfile_frame/lazy.py index 9a82fe17c..a783de032 100644 --- a/flowfile_frame/flowfile_frame/lazy.py +++ b/flowfile_frame/flowfile_frame/lazy.py @@ -171,7 +171,7 @@ def _process_argument(arg: Any, can_be_expr: bool) -> tuple[str, Any, bool, str Tuple of (repr_string, processed_arg_for_polars, convertible_to_code, function_source) """ # Special handling for callables (but not Expr objects which might be callable) - if callable(arg) and not isinstance(arg, (Expr, pl.Expr)) and not hasattr(arg, "expr"): + if callable(arg) and not isinstance(arg, Expr | pl.Expr) and not hasattr(arg, "expr"): return _process_callable_arg(arg) repr_str = _deep_get_repr(arg, can_be_expr) @@ -372,7 +372,7 @@ def _check_for_non_serializable_functions(args: list[Any], kwargs: dict[str, Any def check_value(value: Any, path: str) -> None: """Recursively check for non-serializable functions.""" - if callable(value) and not isinstance(value, (type, pl.Expr)): + if callable(value) and not isinstance(value, type | pl.Expr): # Check if it's a lambda or local function if hasattr(value, "__name__"): if value.__name__ == "": diff --git a/flowfile_frame/flowfile_frame/list_name_space.py b/flowfile_frame/flowfile_frame/list_name_space.py index 605006fe3..dc1c99873 100644 --- a/flowfile_frame/flowfile_frame/list_name_space.py +++ b/flowfile_frame/flowfile_frame/list_name_space.py @@ -160,7 +160,7 @@ def concat(self, other: list[Expr | str] | Expr | str | pl.Series | list[Any]) - other_expr = None # Handle different types of 'other' - if isinstance(other, (Expr, str)): + if isinstance(other, Expr | str): if isinstance(other, Expr): other_expr = other.expr else: @@ -168,7 +168,7 @@ def concat(self, other: list[Expr | str] | Expr | str | pl.Series | list[Any]) - elif isinstance(other, pl.Series): other_expr = pl.lit(other) elif isinstance(other, list): - if len(other) > 0 and isinstance(other[0], (Expr, str, pl.Series)): + if len(other) > 0 and isinstance(other[0], Expr | str | pl.Series): # List of expressions other_expr = [o.expr if hasattr(o, "expr") else (pl.col(o) if isinstance(o, str) else o) for o in other] else: diff --git a/flowfile_frame/flowfile_frame/series.py b/flowfile_frame/flowfile_frame/series.py index e2741acc3..92a4799c6 100644 --- a/flowfile_frame/flowfile_frame/series.py +++ b/flowfile_frame/flowfile_frame/series.py @@ -32,7 +32,7 @@ def __init__( self._name = name.name self._values = name.to_list() self._dtype = name.dtype - elif isinstance(name, (list, tuple)) and values is None: + elif isinstance(name, list | tuple) and values is None: self._s = pl.Series(values=name, dtype=dtype) self._name = "" # Default name is empty string self._values = name diff --git a/flowfile_frame/flowfile_frame/utils.py b/flowfile_frame/flowfile_frame/utils.py index 8b4fa0104..aae434bec 100644 --- a/flowfile_frame/flowfile_frame/utils.py +++ b/flowfile_frame/flowfile_frame/utils.py @@ -12,7 +12,7 @@ def _is_iterable(obj: Any) -> bool: # Avoid treating strings as iterables in this context - return isinstance(obj, Iterable) and not isinstance(obj, (str, bytes)) + return isinstance(obj, Iterable) and not isinstance(obj, str | bytes) def _check_if_convertible_to_code(expressions: list[Any]) -> bool: @@ -77,7 +77,7 @@ def ensure_inputs_as_iterable(inputs: Any | Iterable[Any]) -> list[Any]: if inputs is None or (hasattr(inputs, "__len__") and len(inputs) == 0): return [] # Treat strings/bytes as atomic items, everything else check if iterable - if isinstance(inputs, (str, bytes)) or not _is_iterable(inputs): + if isinstance(inputs, str | bytes) or not _is_iterable(inputs): return [inputs] return list(inputs) @@ -120,7 +120,7 @@ def stringify_values(v: Any) -> str: elif isinstance(v, bool): # Handle booleans explicitly (returns "True" or "False") return str(v) - elif isinstance(v, (int, float, complex, type(None))): + elif isinstance(v, int | float | complex | type(None)): # Handle numbers and None explicitly return str(v) else: diff --git a/flowfile_frame/readme.md b/flowfile_frame/readme.md index 60cc39bee..b6d3c74e2 100644 --- a/flowfile_frame/readme.md +++ b/flowfile_frame/readme.md @@ -86,4 +86,4 @@ open_graph_in_editor(aggregated_df.flow_graph) - **Inspect Data Flow**: See exactly how your data is transformed step by step - **Debugging**: Identify issues in your data pipeline visually -- **Documentation**: Share your data transformation logic with teammates +- **Documentation**: Share your data transformation logic with teammates diff --git a/flowfile_frame/tests/utils.py b/flowfile_frame/tests/utils.py index d11f2d188..28a6bb624 100644 --- a/flowfile_frame/tests/utils.py +++ b/flowfile_frame/tests/utils.py @@ -26,4 +26,3 @@ def find_parent_directory(target_dir_name, start_path=None): current_path = current_path.parent raise FileNotFoundError(f"Directory '{target_dir_name}' not found") - diff --git a/flowfile_frontend/.eslintrc.js b/flowfile_frontend/.eslintrc.js index aa274a43a..efb863129 100644 --- a/flowfile_frontend/.eslintrc.js +++ b/flowfile_frontend/.eslintrc.js @@ -29,6 +29,6 @@ module.exports = { '@typescript-eslint/no-explicit-any': 0, 'vue/multi-word-component-names': 0, 'vue/no-lone-template': 0, - 'linebreak-style': ['error', 'unix'] + 'linebreak-style': ['error', 'unix'] }, -} \ No newline at end of file +} diff --git a/flowfile_frontend/.gitignore b/flowfile_frontend/.gitignore index dd297207f..41ebc4ea1 100644 --- a/flowfile_frontend/.gitignore +++ b/flowfile_frontend/.gitignore @@ -3,4 +3,4 @@ dist build .vscode -.idea \ No newline at end of file +.idea diff --git a/flowfile_frontend/scripts/build.js b/flowfile_frontend/scripts/build.js index 24876822d..e38c47c80 100644 --- a/flowfile_frontend/scripts/build.js +++ b/flowfile_frontend/scripts/build.js @@ -55,4 +55,4 @@ Promise.allSettled([ console.error(Chalk.redBright('Build process failed:'), error); process.exit(1); } -}); \ No newline at end of file +}); diff --git a/flowfile_frontend/scripts/private/tsc.js b/flowfile_frontend/scripts/private/tsc.js index baf0aa1e3..621cce387 100644 --- a/flowfile_frontend/scripts/private/tsc.js +++ b/flowfile_frontend/scripts/private/tsc.js @@ -7,7 +7,7 @@ function compile(directory) { cwd: directory, }); - tscProcess.stdout.on('data', data => + tscProcess.stdout.on('data', data => process.stdout.write(Chalk.yellowBright(`[tsc] `) + Chalk.white(data.toString())) ); diff --git a/flowfile_frontend/src/assets/mac-config/Info.plist b/flowfile_frontend/src/assets/mac-config/Info.plist index 91e954735..a99c2075f 100644 --- a/flowfile_frontend/src/assets/mac-config/Info.plist +++ b/flowfile_frontend/src/assets/mac-config/Info.plist @@ -16,4 +16,4 @@ /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin - \ No newline at end of file + diff --git a/flowfile_frontend/src/assets/mac-config/entitlements.mac.plist b/flowfile_frontend/src/assets/mac-config/entitlements.mac.plist index f0cf1ad18..7f9470056 100644 --- a/flowfile_frontend/src/assets/mac-config/entitlements.mac.plist +++ b/flowfile_frontend/src/assets/mac-config/entitlements.mac.plist @@ -23,4 +23,4 @@ / - \ No newline at end of file + diff --git a/flowfile_frontend/src/main/loading.html b/flowfile_frontend/src/main/loading.html index 794a73de1..6ad6f797d 100644 --- a/flowfile_frontend/src/main/loading.html +++ b/flowfile_frontend/src/main/loading.html @@ -144,21 +144,21 @@

Starting Flowfile

ipcRenderer.on('update-docker-status', (event, { isAvailable, error }) => { const statusEl = document.getElementById('docker-status'); const textEl = document.getElementById('docker-text'); - + statusEl.className = 'status-icon ' + (isAvailable ? 'success' : 'error'); - textEl.textContent = isAvailable ? - 'Docker is available' : + textEl.textContent = isAvailable ? + 'Docker is available' : `Docker unavailable: ${error}`; }); ipcRenderer.on('update-services-status', (event, { status, error }) => { const statusEl = document.getElementById('services-status'); const textEl = document.getElementById('services-text'); - + statusEl.className = 'status-icon ' + (status === 'ready' ? 'success' : error ? 'error' : 'pending'); - textEl.textContent = error || + textEl.textContent = error || (status === 'ready' ? 'Services ready' : 'Starting services...'); }); - \ No newline at end of file + diff --git a/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_reader.svg b/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_reader.svg index 0d644edee..3a927e841 100644 --- a/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_reader.svg +++ b/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_reader.svg @@ -1,24 +1,24 @@ - + - + - + SQL - + - + SOURCE - + diff --git a/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_writer.svg b/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_writer.svg index 34dbfdcb8..95b5c4f30 100644 --- a/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_writer.svg +++ b/flowfile_frontend/src/renderer/app/features/designer/assets/icons/database_writer.svg @@ -1,23 +1,23 @@ - + - + - + SQL - + - + WRITER - + diff --git a/flowfile_frontend/src/renderer/public/images/google.svg b/flowfile_frontend/src/renderer/public/images/google.svg index c599462cb..f6ef21ce9 100644 --- a/flowfile_frontend/src/renderer/public/images/google.svg +++ b/flowfile_frontend/src/renderer/public/images/google.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/flowfile_frontend/src/renderer/public/vite.svg b/flowfile_frontend/src/renderer/public/vite.svg index e7b8dfb1b..ee9fadaf9 100644 --- a/flowfile_frontend/src/renderer/public/vite.svg +++ b/flowfile_frontend/src/renderer/public/vite.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/flowfile_frontend/src/renderer/public/vue.svg b/flowfile_frontend/src/renderer/public/vue.svg index 770e9d333..ca8129c2c 100644 --- a/flowfile_frontend/src/renderer/public/vue.svg +++ b/flowfile_frontend/src/renderer/public/vue.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/flowfile_frontend/src/renderer/styles/components/_column-options.css b/flowfile_frontend/src/renderer/styles/components/_column-options.css index a89ff6488..a50f0ad27 100644 --- a/flowfile_frontend/src/renderer/styles/components/_column-options.css +++ b/flowfile_frontend/src/renderer/styles/components/_column-options.css @@ -3,4 +3,4 @@ overflow-y: auto; padding: 0; margin: 0; - } \ No newline at end of file + } diff --git a/flowfile_frontend/src/renderer/styles/main.css b/flowfile_frontend/src/renderer/styles/main.css index 51131df75..a7466d53a 100644 --- a/flowfile_frontend/src/renderer/styles/main.css +++ b/flowfile_frontend/src/renderer/styles/main.css @@ -605,4 +605,3 @@ button { .cm-panels-bottom { border-top-color: var(--color-border-primary) !important; } - diff --git a/flowfile_frontend/src/renderer/styles/utils/_layout.css b/flowfile_frontend/src/renderer/styles/utils/_layout.css index e11e191a6..3dd1b1816 100644 --- a/flowfile_frontend/src/renderer/styles/utils/_layout.css +++ b/flowfile_frontend/src/renderer/styles/utils/_layout.css @@ -29,4 +29,4 @@ .gap-3 { gap: var(--spacing-md); } /* Size utilities */ -.w-full { width: 100%; } \ No newline at end of file +.w-full { width: 100%; } diff --git a/flowfile_frontend/src/renderer/styles/utils/_spacing.css b/flowfile_frontend/src/renderer/styles/utils/_spacing.css index eee85565b..8b16382ae 100644 --- a/flowfile_frontend/src/renderer/styles/utils/_spacing.css +++ b/flowfile_frontend/src/renderer/styles/utils/_spacing.css @@ -1,4 +1,4 @@ .p-default { margin: var(--spacing-xs); margin-bottom: var(--spacing-sm); - } \ No newline at end of file + } diff --git a/flowfile_frontend/tests/app.spec.ts b/flowfile_frontend/tests/app.spec.ts index 940ae2650..36ce02241 100644 --- a/flowfile_frontend/tests/app.spec.ts +++ b/flowfile_frontend/tests/app.spec.ts @@ -1,6 +1,6 @@ import { test, expect } from '@playwright/test'; import { launchElectronApp, closeElectronApp } from './helpers/electronTestHelper'; -import { ElectronApplication } from 'playwright-core'; +import { ElectronApplication } from 'playwright-core'; let electronApp: ElectronApplication | undefined; @@ -24,15 +24,15 @@ test.describe('Services Startup Tests', () => { test('app should load properly and respond to navigation', async () => { test.skip(!electronApp, 'Electron app failed to launch'); - + try { const mainWindow = await electronApp!.firstWindow(); expect(mainWindow).toBeDefined(); - + const isWindowAvailable = await mainWindow.evaluate(() => { return true; }).catch(() => false); - + if (!isWindowAvailable) { console.log('Window is no longer available, skipping navigation test'); test.skip(); @@ -40,42 +40,42 @@ test.describe('Services Startup Tests', () => { } console.log('Waiting for app to fully initialize...'); - + for (let i = 0; i < 10; i++) { const isStillAvailable = await mainWindow.evaluate(() => { return true; }).catch(() => false); - + if (!isStillAvailable) { console.log('Window closed during initialization wait, skipping test'); test.skip(); return; } - + await new Promise(r => setTimeout(r, 500)); } - + try { const canNavigate = await mainWindow.evaluate(() => { return true; }).catch(() => false); - + if (!canNavigate) { console.log('Window closed before navigation, skipping'); test.skip(); return; } - + console.log('Attempting to navigate within the app...'); await mainWindow.evaluate(() => { console.log('Current URL:', window.location.href); - + if (window.history) { window.history.pushState({}, '', '/dashboard'); console.log('Navigated to /dashboard'); } }); - + const canWait = await mainWindow.evaluate(() => true).catch(() => false); if (canWait) { await new Promise(r => setTimeout(r, 1000)); @@ -83,17 +83,17 @@ test.describe('Services Startup Tests', () => { } catch (navError) { console.warn('Navigation attempt failed:', navError); } - + console.log('Navigation test completed successfully'); - + } catch (error) { if (error.message.includes('Target page, context or browser has been closed')) { console.log('Window was closed unexpectedly, marking test as passed anyway'); return; } - + console.error("Error in navigation test:", error); throw error; } }); -}); \ No newline at end of file +}); diff --git a/flowfile_frontend/tests/helpers/electronTestHelper.ts b/flowfile_frontend/tests/helpers/electronTestHelper.ts index 3513a7eb0..052da27be 100644 --- a/flowfile_frontend/tests/helpers/electronTestHelper.ts +++ b/flowfile_frontend/tests/helpers/electronTestHelper.ts @@ -28,10 +28,10 @@ export async function launchElectronApp(): Promise { }, SERVICES_STARTUP_TIMEOUT); const checkComplete = () => { - const isComplete = process.platform === 'win32' + const isComplete = process.platform === 'win32' ? (servicesStarted && startupReceived && electronApp) : (servicesStarted && startupReceived && windowReady && electronApp); - + if (isComplete) { clearTimeout(timeout); resolve(electronApp!); @@ -61,7 +61,7 @@ export async function launchElectronApp(): Promise { startupReceived = true; checkComplete(); } - + if (text.includes('Window ready to show')) { console.log('Detected window ready'); windowReady = true; @@ -112,7 +112,7 @@ export async function closeElectronApp(app: ElectronApplication | undefined): Pr mainWindow = await app.firstWindow().catch(() => null); if (mainWindow) { console.log('Giving app chance to clean up resources...'); - + try { await mainWindow.evaluate(() => { const api = (window as any).electronAPI; @@ -145,4 +145,4 @@ export async function closeElectronApp(app: ElectronApplication | undefined): Pr } catch (error) { console.error('Error in closeElectronApp:', error); } -} \ No newline at end of file +} diff --git a/flowfile_frontend/types/preload.d.ts b/flowfile_frontend/types/preload.d.ts index 104284a9f..57970154a 100644 --- a/flowfile_frontend/types/preload.d.ts +++ b/flowfile_frontend/types/preload.d.ts @@ -6,12 +6,11 @@ interface ElectronAPI { onDockerStatusUpdate: (callback: (status: any) => void) => () => void; onServicesStatusUpdate: (callback: (status: any) => void) => () => void; } - + declare global { interface Window { electronAPI: ElectronAPI; } } - + export {}; - \ No newline at end of file diff --git a/flowfile_worker/Dockerfile b/flowfile_worker/Dockerfile index 5487fa13e..b5367c2dd 100644 --- a/flowfile_worker/Dockerfile +++ b/flowfile_worker/Dockerfile @@ -47,4 +47,4 @@ ENV FLOWFILE_MODE=docker WORKDIR /app # Command to run the application -CMD ["python", "-m", "flowfile_worker.main"] \ No newline at end of file +CMD ["python", "-m", "flowfile_worker.main"] diff --git a/flowfile_worker/README.md b/flowfile_worker/README.md index 8478213ba..2872336d5 100644 --- a/flowfile_worker/README.md +++ b/flowfile_worker/README.md @@ -75,4 +75,4 @@ The worker is designed to run in a trusted environment. When exposing to externa ## 📝 License -[MIT License](LICENSE) \ No newline at end of file +[MIT License](LICENSE) diff --git a/flowfile_worker/flowfile_worker/__init__.py b/flowfile_worker/flowfile_worker/__init__.py index a7075c98d..7c16614ca 100644 --- a/flowfile_worker/flowfile_worker/__init__.py +++ b/flowfile_worker/flowfile_worker/__init__.py @@ -10,7 +10,7 @@ __version__ = version("Flowfile") except PackageNotFoundError: __version__ = "0.5.0" -multiprocessing.set_start_method('spawn', force=True) +multiprocessing.set_start_method("spawn", force=True) from multiprocessing import get_context diff --git a/flowfile_worker/flowfile_worker/create/utils.py b/flowfile_worker/flowfile_worker/create/utils.py index 6260062bd..0decd3a1f 100644 --- a/flowfile_worker/flowfile_worker/create/utils.py +++ b/flowfile_worker/flowfile_worker/create/utils.py @@ -34,7 +34,7 @@ def generate_phone_number(): return fake.phone_number() data = [] - for i in range(n_records): + for _i in range(n_records): name = generate_name() data.append( dict( diff --git a/flowfile_worker/flowfile_worker/funcs.py b/flowfile_worker/flowfile_worker/funcs.py index b8151cb23..15269315d 100644 --- a/flowfile_worker/flowfile_worker/funcs.py +++ b/flowfile_worker/flowfile_worker/funcs.py @@ -164,7 +164,7 @@ def calculate_schema_logic( include_header=True, header_name="column_name", column_names=stats_headers ).to_dicts() } - for i, (col_stat, n_unique_values) in enumerate(zip(stats.values(), n_unique_per_cols, strict=False)): + for _i, (col_stat, n_unique_values) in enumerate(zip(stats.values(), n_unique_per_cols, strict=False)): col_stat["n_unique"] = n_unique_values col_stat["examples"] = ", ".join({str(col_stat["min"]), str(col_stat["max"])}) col_stat["null_count"] = int(float(col_stat["null_count"])) diff --git a/flowfile_worker/tests/external_sources/test_sql_source.py b/flowfile_worker/tests/external_sources/test_sql_source.py index d3d3d4762..d988e744b 100644 --- a/flowfile_worker/tests/external_sources/test_sql_source.py +++ b/flowfile_worker/tests/external_sources/test_sql_source.py @@ -70,5 +70,3 @@ def test_write_serialized_df_to_database(pw): database_read_settings = DatabaseReadSettings(connection=database_connection, query='SELECT * FROM public.test_output') result_df = read_sql_source(database_read_settings) assert df.equals(result_df), "DataFrame written to the database should match the original DataFrame" - - diff --git a/poetry.lock b/poetry.lock index 49e717eea..72c908688 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiobotocore" @@ -6,6 +6,7 @@ version = "2.23.1" description = "Async client for aws services using botocore and aiohttp" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiobotocore-2.23.1-py3-none-any.whl", hash = "sha256:d81c54d2eae2406ea9a473fea518fed580cf37bc4fc51ce43ba81546e5305114"}, {file = "aiobotocore-2.23.1.tar.gz", hash = "sha256:a59f2a78629b97d52f10936b79c73de64e481a8c44a62c1871f088df6c1afc4f"}, @@ -31,6 +32,7 @@ version = "24.1.0" description = "File support for asyncio." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5"}, {file = "aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c"}, @@ -42,6 +44,7 @@ version = "2.6.1" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8"}, {file = "aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558"}, @@ -53,6 +56,7 @@ version = "3.13.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7"}, {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821"}, @@ -187,7 +191,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2)", "aiodns (>=3.3.0)", "backports.zstd", "brotlicffi (>=1.2)"] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aioitertools" @@ -195,6 +199,7 @@ version = "0.13.0" description = "itertools and builtins for AsyncIO and mixed iterables" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be"}, {file = "aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c"}, @@ -206,6 +211,7 @@ version = "1.4.0" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, @@ -221,6 +227,7 @@ version = "0.17.5" description = "Python graph (network) package" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "altgraph-0.17.5-py2.py3-none-any.whl", hash = "sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597"}, {file = "altgraph-0.17.5.tar.gz", hash = "sha256:c87b395dd12fabde9c99573a9749d67da8d29ef9de0125c7f536699b4a9bc9e7"}, @@ -232,6 +239,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -243,6 +251,7 @@ version = "4.12.0" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb"}, {file = "anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0"}, @@ -254,7 +263,7 @@ idna = ">=2.8" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -trio = ["trio (>=0.31.0)", "trio (>=0.32.0)"] +trio = ["trio (>=0.31.0) ; python_version < \"3.10\"", "trio (>=0.32.0) ; python_version >= \"3.10\""] [[package]] name = "arro3-core" @@ -262,6 +271,7 @@ version = "0.6.5" description = "" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae"}, {file = "arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a"}, @@ -331,7 +341,7 @@ files = [ ] [package.dependencies] -typing-extensions = {version = "*", markers = "python_full_version < \"3.12\""} +typing-extensions = {version = "*", markers = "python_full_version < \"3.12.0\""} [[package]] name = "async-timeout" @@ -339,6 +349,8 @@ version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -350,6 +362,7 @@ version = "25.4.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, @@ -361,13 +374,14 @@ version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] [[package]] name = "backrefs" @@ -375,6 +389,7 @@ version = "6.1" description = "A wrapper around re and regex that adds additional back references." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "backrefs-6.1-py310-none-any.whl", hash = "sha256:2a2ccb96302337ce61ee4717ceacfbf26ba4efb1d55af86564b8bbaeda39cac1"}, {file = "backrefs-6.1-py311-none-any.whl", hash = "sha256:e82bba3875ee4430f4de4b6db19429a27275d95a5f3773c57e9e18abc23fd2b7"}, @@ -394,6 +409,7 @@ version = "4.3.0" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, @@ -458,6 +474,7 @@ version = "1.38.46" description = "The AWS SDK for Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "boto3-1.38.46-py3-none-any.whl", hash = "sha256:9c8e88a32a6465e5905308708cff5b17547117f06982908bdfdb0108b4a65079"}, {file = "boto3-1.38.46.tar.gz", hash = "sha256:d1ca2b53138afd0341e1962bd52be6071ab7a63c5b4f89228c5ef8942c40c852"}, @@ -477,6 +494,7 @@ version = "1.38.46" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "botocore-1.38.46-py3-none-any.whl", hash = "sha256:89ca782ffbf2e8769ca9c89234cfa5ca577f1987d07d913ee3c68c4776b1eb5b"}, {file = "botocore-1.38.46.tar.gz", hash = "sha256:8798e5a418c27cf93195b077153644aea44cb171fcd56edc1ecebaa1e49e226e"}, @@ -496,6 +514,7 @@ version = "5.5.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, @@ -507,6 +526,7 @@ version = "2026.1.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c"}, {file = "certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120"}, @@ -518,6 +538,8 @@ version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -608,12 +630,25 @@ files = [ [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} +[[package]] +name = "cfgv" +version = "3.5.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0"}, + {file = "cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132"}, +] + [[package]] name = "charset-normalizer" version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, @@ -736,6 +771,7 @@ version = "8.3.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -750,6 +786,7 @@ version = "3.1.2" description = "Pickler class to extend the standard pickle.Pickler functionality" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a"}, {file = "cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414"}, @@ -761,10 +798,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\""} [[package]] name = "connectorx" @@ -772,6 +811,7 @@ version = "0.4.4" description = "" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "connectorx-0.4.4-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:58a3f9e05a42066dd8e2f1da6c9bbd18662817eba7968eb88031f4b3365831e0"}, {file = "connectorx-0.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcaeb9ccede6bb63e6cc85b8004aef08a2ec4cd128df18390da46bdb2daa378b"}, @@ -801,6 +841,7 @@ version = "45.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.7" +groups = ["main"] files = [ {file = "cryptography-45.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3be4f21c6245930688bd9e162829480de027f8bf962ede33d4f8ba7d67a00cee"}, {file = "cryptography-45.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67285f8a611b0ebc0857ced2081e30302909f571a46bfa7a3cc0ad303fe015c6"}, @@ -845,10 +886,10 @@ files = [ cffi = {version = ">=1.14", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs ; python_full_version >= \"3.8.0\"", "sphinx-rtd-theme (>=3.0.0) ; python_full_version >= \"3.8.0\""] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_full_version >= \"3.8.0\""] +pep8test = ["check-sdist ; python_full_version >= \"3.8.0\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==45.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -860,6 +901,7 @@ version = "0.9.0" description = "Async database support for Python." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "databases-0.9.0-py3-none-any.whl", hash = "sha256:9ee657c9863b34f8d3a06c06eafbe1bda68af2a434b56996312edf1f1c0b6297"}, {file = "databases-0.9.0.tar.gz", hash = "sha256:d2f259677609bf187737644c95fa41701072e995dfeb8d2882f335795c5b61b0"}, @@ -884,6 +926,7 @@ version = "1.3.0" description = "Native Delta Lake Python binding based on delta-rs with Pandas integration" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "deltalake-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7381ed01f5968c4befdb6bc8706d99b39f33722074d7ee3be08e488aca3f1681"}, {file = "deltalake-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:66a96bc03f57b5868817d185f4d1148f43162b0bec8bc748a2eb8f75a8a5fca8"}, @@ -907,6 +950,7 @@ version = "1.3.1" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] files = [ {file = "deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f"}, {file = "deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223"}, @@ -916,7 +960,19 @@ files = [ wrapt = ">=1.10,<3" [package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] + +[[package]] +name = "distlib" +version = "0.4.0" +description = "Distribution utilities" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, + {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, +] [[package]] name = "docker" @@ -924,6 +980,7 @@ version = "7.1.0" description = "A Python library for the Docker Engine API." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, @@ -946,6 +1003,7 @@ version = "0.19.1" description = "ECDSA cryptographic signature library (pure python)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.6" +groups = ["main"] files = [ {file = "ecdsa-0.19.1-py2.py3-none-any.whl", hash = "sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3"}, {file = "ecdsa-0.19.1.tar.gz", hash = "sha256:478cba7b62555866fcb3bb3fe985e06decbdb68ef55713c4e5ab98c57d508e61"}, @@ -964,6 +1022,7 @@ version = "2.0.0" description = "An implementation of lxml.xmlfile for the standard library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, @@ -975,6 +1034,8 @@ version = "1.3.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, @@ -992,6 +1053,7 @@ version = "23.1.0" description = "Faker is a Python package that generates fake data for you." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "Faker-23.1.0-py3-none-any.whl", hash = "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2"}, {file = "Faker-23.1.0.tar.gz", hash = "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b"}, @@ -1006,6 +1068,7 @@ version = "0.115.14" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca"}, {file = "fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739"}, @@ -1026,6 +1089,7 @@ version = "0.12.1" description = "A fast excel file reader for Python, written in Rust" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fastexcel-0.12.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7c4c959a49329a53540ed86d4f92cd4af2d774b08c47ad190841e4daf042758a"}, {file = "fastexcel-0.12.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e889290b2d1437e57c4fb500657f0d2908de6aa5b291475b93e3b5d7e0cea938"}, @@ -1042,12 +1106,25 @@ pyarrow = ">=8.0.0" pandas = ["pandas (>=1.4.4)"] polars = ["polars (>=0.16.14)"] +[[package]] +name = "filelock" +version = "3.20.2" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "filelock-3.20.2-py3-none-any.whl", hash = "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8"}, + {file = "filelock-3.20.2.tar.gz", hash = "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64"}, +] + [[package]] name = "frozenlist" version = "1.8.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, @@ -1187,6 +1264,7 @@ version = "2025.12.0" description = "File-system specification" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b"}, {file = "fsspec-2025.12.0.tar.gz", hash = "sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973"}, @@ -1217,7 +1295,7 @@ smb = ["smbprotocol"] ssh = ["paramiko"] test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard ; python_version < \"3.14\""] tqdm = ["tqdm"] [[package]] @@ -1226,6 +1304,7 @@ version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, @@ -1243,6 +1322,8 @@ version = "3.3.0" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] +markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" files = [ {file = "greenlet-3.3.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f8496d434d5cb2dce025773ba5597f71f5410ae499d5dd9533e0653258cdb3d"}, {file = "greenlet-3.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b96dc7eef78fd404e022e165ec55327f935b9b52ff355b067eb4a0267fc1cffb"}, @@ -1304,6 +1385,7 @@ version = "1.15.0" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "griffe-1.15.0-py3-none-any.whl", hash = "sha256:6f6762661949411031f5fcda9593f586e6ce8340f0ba88921a0f2ef7a81eb9a3"}, {file = "griffe-1.15.0.tar.gz", hash = "sha256:7726e3afd6f298fbc3696e67958803e7ac843c1cfe59734b6251a40cdbfb5eea"}, @@ -1321,6 +1403,7 @@ version = "1.1.8" description = "Griffe extension for Pydantic." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "griffe_pydantic-1.1.8-py3-none-any.whl", hash = "sha256:22212c94216e03bf43d30ff3bc79cd53fb973ae2fe81d8b7510242232a1e6764"}, {file = "griffe_pydantic-1.1.8.tar.gz", hash = "sha256:72cde69c74c70f3dc0385a7a5243c736cd6bf6fcf8a41cae497383defe107041"}, @@ -1335,6 +1418,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -1346,6 +1430,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -1367,6 +1452,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1379,18 +1465,34 @@ httpcore = "==1.*" idna = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "identify" +version = "2.6.15" +description = "File identification library for Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757"}, + {file = "identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf"}, +] + +[package.extras] +license = ["ukkonen"] + [[package]] name = "idna" version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -1405,6 +1507,7 @@ version = "2.3.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, @@ -1416,6 +1519,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -1433,6 +1537,7 @@ version = "1.0.1" description = "JSON Matching Expressions" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -1444,6 +1549,7 @@ version = "3.4.1" description = "A robust implementation of concurrent.futures.ProcessPoolExecutor" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "loky-3.4.1-py3-none-any.whl", hash = "sha256:7132da80d1a057b5917ff32c7867b65ed164aae84c259a1dbc44375791280c87"}, {file = "loky-3.4.1.tar.gz", hash = "sha256:66db350de68c301299c882ace3b8f06ba5c4cb2c45f8fcffd498160ce8280753"}, @@ -1458,6 +1564,8 @@ version = "1.16.4" description = "Mach-O header analysis and editing" optional = false python-versions = "*" +groups = ["main"] +markers = "sys_platform == \"darwin\"" files = [ {file = "macholib-1.16.4-py2.py3-none-any.whl", hash = "sha256:da1a3fa8266e30f0ce7e97c6a54eefaae8edd1e5f86f3eb8b95457cae90265ea"}, {file = "macholib-1.16.4.tar.gz", hash = "sha256:f408c93ab2e995cd2c46e34fe328b130404be143469e41bc366c807448979362"}, @@ -1472,6 +1580,7 @@ version = "3.10" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c"}, {file = "markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e"}, @@ -1487,6 +1596,7 @@ version = "4.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, @@ -1510,6 +1620,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -1608,6 +1719,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1619,6 +1731,7 @@ version = "1.3.4" description = "A deep merge function for 🐍." optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, @@ -1630,6 +1743,7 @@ version = "0.4.7" description = "Expand standard functools to methods" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "methodtools-0.4.7-py2.py3-none-any.whl", hash = "sha256:5e188c780b236adc12e75b5f078c5afb419ef99eb648569fc6d7071f053a1f11"}, {file = "methodtools-0.4.7.tar.gz", hash = "sha256:e213439dd64cfe60213f7015da6efe5dd4003fd89376db3baa09fe13ec2bb0ba"}, @@ -1640,7 +1754,7 @@ wirerope = ">=0.4.7" [package.extras] doc = ["sphinx"] -test = ["functools32 (>=3.2.3-2)", "pytest (>=4.6.7)", "pytest-cov (>=2.6.1)"] +test = ["functools32 (>=3.2.3-2) ; python_version < \"3\"", "pytest (>=4.6.7)", "pytest-cov (>=2.6.1)"] [[package]] name = "mkdocs" @@ -1648,6 +1762,7 @@ version = "1.6.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, @@ -1670,7 +1785,7 @@ watchdog = ">=2.0" [package.extras] i18n = ["babel (>=2.9.0)"] -min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.4)", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform_system == \"Windows\"", "ghp-import (==1.0)", "importlib-metadata (==4.4) ; python_version < \"3.10\"", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] [[package]] name = "mkdocs-autorefs" @@ -1678,6 +1793,7 @@ version = "1.4.3" description = "Automatically link across pages in MkDocs." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "mkdocs_autorefs-1.4.3-py3-none-any.whl", hash = "sha256:469d85eb3114801d08e9cc55d102b3ba65917a869b893403b8987b601cf55dc9"}, {file = "mkdocs_autorefs-1.4.3.tar.gz", hash = "sha256:beee715b254455c4aa93b6ef3c67579c399ca092259cc41b7d9342573ff1fc75"}, @@ -1694,6 +1810,7 @@ version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"}, {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"}, @@ -1710,6 +1827,7 @@ version = "9.7.1" description = "Documentation that simply works" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_material-9.7.1-py3-none-any.whl", hash = "sha256:3f6100937d7d731f87f1e3e3b021c97f7239666b9ba1151ab476cabb96c60d5c"}, {file = "mkdocs_material-9.7.1.tar.gz", hash = "sha256:89601b8f2c3e6c6ee0a918cc3566cb201d40bf37c3cd3c2067e26fadb8cce2b8"}, @@ -1739,6 +1857,7 @@ version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, @@ -1750,6 +1869,7 @@ version = "0.30.1" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "mkdocstrings-0.30.1-py3-none-any.whl", hash = "sha256:41bd71f284ca4d44a668816193e4025c950b002252081e387433656ae9a70a82"}, {file = "mkdocstrings-0.30.1.tar.gz", hash = "sha256:84a007aae9b707fb0aebfc9da23db4b26fc9ab562eb56e335e9ec480cb19744f"}, @@ -1774,6 +1894,7 @@ version = "1.19.0" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "mkdocstrings_python-1.19.0-py3-none-any.whl", hash = "sha256:395c1032af8f005234170575cc0c5d4d20980846623b623b35594281be4a3059"}, {file = "mkdocstrings_python-1.19.0.tar.gz", hash = "sha256:917aac66cf121243c11db5b89f66b0ded6c53ec0de5318ff5e22424eb2f2e57c"}, @@ -1791,6 +1912,7 @@ version = "5.2.0" description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "mmh3-5.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:81c504ad11c588c8629536b032940f2a359dda3b6cbfd4ad8f74cb24dcd1b0bc"}, {file = "mmh3-5.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0b898cecff57442724a0f52bf42c2de42de63083a91008fb452887e372f9c328"}, @@ -1929,6 +2051,7 @@ version = "6.7.0" description = "multidict implementation" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "multidict-6.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9f474ad5acda359c8758c8accc22032c6abe6dc87a8be2440d097785e27a9349"}, {file = "multidict-6.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b7a9db5a870f780220e931d0002bbfd88fb53aceb6293251e2c839415c1b20e"}, @@ -2081,12 +2204,25 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "nodeenv" +version = "1.10.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +files = [ + {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, + {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, +] + [[package]] name = "numpy" version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -2132,6 +2268,7 @@ version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, @@ -2146,6 +2283,7 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -2157,6 +2295,7 @@ version = "0.5.7" description = "Divides large result sets into pages for easier browsing" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, @@ -2172,6 +2311,7 @@ version = "2.3.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, @@ -2271,6 +2411,7 @@ version = "1.7.4" description = "comprehensive password hashing framework supporting over 30 schemes" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "passlib-1.7.4-py2.py3-none-any.whl", hash = "sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1"}, {file = "passlib-1.7.4.tar.gz", hash = "sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04"}, @@ -2288,6 +2429,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -2299,6 +2441,8 @@ version = "2024.8.26" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, @@ -2310,6 +2454,8 @@ version = "2.1.2" description = "Python datetimes made easy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, @@ -2344,6 +2490,7 @@ version = "0.4.0" description = "Efficient fuzzy matching for Polars DataFrames with support for multiple string similarity algorithms" optional = false python-versions = "<4.0,>=3.10" +groups = ["main"] files = [ {file = "pl_fuzzy_frame_match-0.4.0-py3-none-any.whl", hash = "sha256:7d35b4661fca2bf20afa4141baf619473fd455d5c8d3a1c44030120ade091cd9"}, {file = "pl_fuzzy_frame_match-0.4.0.tar.gz", hash = "sha256:157631fc6e1e2cd3dd797a335af8177e5e22d0edf719a0aa84ff99edf792c1c9"}, @@ -2363,6 +2510,7 @@ version = "4.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31"}, {file = "platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda"}, @@ -2379,6 +2527,7 @@ version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -2394,6 +2543,8 @@ version = "1.25.2" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "polars-1.25.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59f2a34520ea4307a22e18b832310f8045a8a348606ca99ae785499b31eb4170"}, {file = "polars-1.25.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9fe45bdc2327c2e2b64e8849a992b6d3bd4a7e7848b8a7a3a439cca9674dc87"}, @@ -2430,7 +2581,7 @@ pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] @@ -2440,6 +2591,8 @@ version = "1.31.0" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "sys_platform != \"win32\"" files = [ {file = "polars-1.31.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ccc68cd6877deecd46b13cbd2663ca89ab2a2cb1fe49d5cfc66a9cef166566d9"}, {file = "polars-1.31.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:a94c5550df397ad3c2d6adc212e59fd93d9b044ec974dd3653e121e6487a7d21"}, @@ -2476,7 +2629,7 @@ pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] @@ -2486,6 +2639,7 @@ version = "0.4.3" description = "Polars plugin for pairwise distance functions" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "polars_distance-0.4.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d9a388510cad81be5c7ba978720595369a530a52394322f74e2455b9591ea73f"}, {file = "polars_distance-0.4.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:1aa81c2738825844bac342a7de33b4d940140c964f0e587118aab4f87674b02d"}, @@ -2509,6 +2663,7 @@ version = "0.10.4" description = "" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "polars_ds-0.10.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6f6dde4e835129155d0386fcdd0aa6b603e305eaf60c78d7ee743c4897bb1547"}, {file = "polars_ds-0.10.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1715b9aaef3e8f119eaf8d37cdc921c96350237edd1f3263d2d6e699b27c3e48"}, @@ -2534,6 +2689,7 @@ version = "0.4.10.0" description = "Transform string-based expressions into Polars DataFrame operations" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "polars_expr_transformer-0.4.10.0-py3-none-any.whl", hash = "sha256:0f811839218beace53e7556ea79a9c23797af5608d70b0561dafb53f6d63e727"}, {file = "polars_expr_transformer-0.4.10.0.tar.gz", hash = "sha256:708f9588e5736443f5dcd35ae37114898077533b5ce87a032edab753d060d8f5"}, @@ -2550,6 +2706,7 @@ version = "0.3.0" description = "High-performance graph analysis and pattern mining extension for Polars" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "polars_grouper-0.3.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6a2c56eb4621502447268c2d40bfc7696fe291691fe777b257cdda869bfbdde2"}, {file = "polars_grouper-0.3.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:3701fea159f2104d78e8aaad65c2af698275a8b8aa036a8c1d98ef18de06a822"}, @@ -2569,6 +2726,7 @@ version = "0.3.4" description = "Fast similarity join for polars DataFrames. Fork by Edwardvaneechoud with fixes." optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "polars_simed-0.3.4-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:fd8adf37248e19397f2ac9772d8bda49b67e2e22b3209c6b2decba181addede3"}, {file = "polars_simed-0.3.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e6755e2d895efdaf65d1948cf22999e6d9afc488b2f31a5c47850d89266c4b23"}, @@ -2581,12 +2739,32 @@ files = [ [package.dependencies] polars = {version = ">=1.8.2", extras = ["pyarrow"]} +[[package]] +name = "pre-commit" +version = "4.5.1" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77"}, + {file = "pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + [[package]] name = "propcache" version = "0.4.1" description = "Accelerated property cache" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db"}, {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8"}, @@ -2718,6 +2896,7 @@ version = "2.9.11" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6fe6b47d0b42ce1c9f1fa3e35bb365011ca22e39db37074458f27921dca40f2"}, @@ -2725,8 +2904,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c47676e5b485393f069b4d7a811267d3168ce46f988fa602658b8bb901e9e64d"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a28d8c01a7b27a1e3265b11250ba7557e5f72b5ee9e5f3a2fa8d2949c29bf5d2"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f3f2732cf504a1aa9e9609d02f79bea1067d99edf844ab92c247bbca143303b"}, + {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:865f9945ed1b3950d968ec4690ce68c55019d79e4497366d36e090327ce7db14"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91537a8df2bde69b1c1db01d6d944c831ca793952e4f57892600e96cee95f2cd"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4dca1f356a67ecb68c81a7bc7809f1569ad9e152ce7fd02c2f2036862ca9f66b"}, + {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0da4de5c1ac69d94ed4364b6cbe7190c1a70d325f112ba783d83f8440285f152"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37d8412565a7267f7d79e29ab66876e55cb5e8e7b3bbf94f8206f6795f8f7e7e"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:c665f01ec8ab273a61c62beeb8cce3014c214429ced8a308ca1fc410ecac3a39"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e8480afd62362d0a6a27dd09e4ca2def6fa50ed3a4e7c09165266106b2ffa10"}, @@ -2734,8 +2915,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e164359396576a3cc701ba8af4751ae68a07235d7a380c631184a611220d9a4"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d57c9c387660b8893093459738b6abddbb30a7eab058b77b0d0d1c7d521ddfd7"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c226ef95eb2250974bf6fa7a842082b31f68385c4f3268370e3f3870e7859ee"}, + {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a311f1edc9967723d3511ea7d2708e2c3592e3405677bf53d5c7246753591fbb"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb415404821b6d1c47353ebe9c8645967a5235e6d88f914147e7fd411419e6f"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f07c9c4a5093258a03b28fab9b4f151aa376989e7f35f855088234e656ee6a94"}, + {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:00ce1830d971f43b667abe4a56e42c1e2d594b32da4802e44a73bacacb25535f"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cffe9d7697ae7456649617e8bb8d7a45afb71cd13f7ab22af3e5c61f04840908"}, {file = "psycopg2_binary-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:304fd7b7f97eef30e91b8f7e720b3db75fee010b520e434ea35ed1ff22501d03"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4"}, @@ -2743,8 +2926,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db"}, + {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a"}, + {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d"}, {file = "psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c"}, @@ -2752,8 +2937,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3"}, + {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c"}, + {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1"}, {file = "psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1"}, @@ -2761,8 +2948,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c"}, + {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f"}, + {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d"}, {file = "psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:20e7fb94e20b03dcc783f76c0865f9da39559dcc0c28dd1a3fce0d01902a6b9c"}, @@ -2770,8 +2959,10 @@ files = [ {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9d3a9edcfbe77a3ed4bc72836d466dfce4174beb79eda79ea155cc77237ed9e8"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:44fc5c2b8fa871ce7f0023f619f1349a0aa03a0857f2c96fbc01c657dcbbdb49"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9c55460033867b4622cda1b6872edf445809535144152e5d14941ef591980edf"}, + {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2d11098a83cca92deaeaed3d58cfd150d49b3b06ee0d0852be466bf87596899e"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:691c807d94aecfbc76a14e1408847d59ff5b5906a04a23e12a89007672b9e819"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:8b81627b691f29c4c30a8f322546ad039c40c328373b11dff7490a3e1b517855"}, + {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:b637d6d941209e8d96a072d7977238eea128046effbf37d1d8b2c0764750017d"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:41360b01c140c2a03d346cec3280cf8a71aa07d94f3b1509fa0161c366af66b4"}, {file = "psycopg2_binary-2.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:875039274f8a2361e5207857899706da840768e2a775bf8c65e82f60b197df02"}, ] @@ -2782,6 +2973,7 @@ version = "18.1.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c"}, {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4"}, @@ -2836,6 +3028,7 @@ version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, @@ -2847,6 +3040,8 @@ version = "2.23" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, @@ -2858,6 +3053,7 @@ version = "2.9.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, @@ -2873,7 +3069,7 @@ typing-extensions = [ [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and sys_platform == \"win32\""] [[package]] name = "pydantic-core" @@ -2881,6 +3077,7 @@ version = "2.23.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, @@ -2982,6 +3179,7 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -2996,6 +3194,7 @@ version = "0.9.1" description = "Apache Iceberg is an open table format for huge analytic datasets" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,!=3.8.*,>=3.9" +groups = ["main"] files = [ {file = "pyiceberg-0.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a183d9217eb82159c01b23c683057f96c8b2375f592b921721d1c157895e2df"}, {file = "pyiceberg-0.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:57030bb15c397b0379242907c5611f5b4338fb799e972353fd0edafde6cfd2ef"}, @@ -3054,7 +3253,7 @@ pandas = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)"] polars = ["polars (>=1.21.0,<2.0.0)"] pyarrow = ["pyarrow (>=17.0.0,<20.0.0)"] pyiceberg-core = ["pyiceberg-core (>=0.4.0,<0.5.0)"] -ray = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)", "ray (==2.10.0)", "ray (>=2.10.0,<3.0.0)"] +ray = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)", "ray (==2.10.0) ; python_version < \"3.9\"", "ray (>=2.10.0,<3.0.0) ; python_version >= \"3.9\""] rest-sigv4 = ["boto3 (>=1.24.59)"] s3fs = ["s3fs (>=2023.1.0)"] snappy = ["python-snappy (>=0.6.0,<1.0.0)"] @@ -3068,6 +3267,7 @@ version = "6.17.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false python-versions = "<3.15,>=3.8" +groups = ["main"] files = [ {file = "pyinstaller-6.17.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:4e446b8030c6e5a2f712e3f82011ecf6c7ead86008357b0d23a0ec4bcde31dac"}, {file = "pyinstaller-6.17.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:aa9fd87aaa28239c6f0d0210114029bd03f8cac316a90bab071a5092d7c85ad7"}, @@ -3102,6 +3302,7 @@ version = "2025.11" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyinstaller_hooks_contrib-2025.11-py3-none-any.whl", hash = "sha256:777e163e2942474aa41a8e6d31ac1635292d63422c3646c176d584d04d971c34"}, {file = "pyinstaller_hooks_contrib-2025.11.tar.gz", hash = "sha256:dfe18632e06655fa88d218e0d768fd753e1886465c12a6d4bce04f1aaeec917d"}, @@ -3117,6 +3318,7 @@ version = "10.20" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pymdown_extensions-10.20-py3-none-any.whl", hash = "sha256:ea9e62add865da80a271d00bfa1c0fa085b20d133fb3fc97afdc88e682f60b2f"}, {file = "pymdown_extensions-10.20.tar.gz", hash = "sha256:5c73566ab0cf38c6ba084cb7c5ea64a119ae0500cce754ccb682761dfea13a52"}, @@ -3135,6 +3337,7 @@ version = "3.3.1" description = "pyparsing - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pyparsing-3.3.1-py3-none-any.whl", hash = "sha256:023b5e7e5520ad96642e2c6db4cb683d3970bd640cdf7115049a6e9c3682df82"}, {file = "pyparsing-3.3.1.tar.gz", hash = "sha256:47fad0f17ac1e2cad3de3b458570fbc9b03560aa029ed5e16ee5554da9a2251c"}, @@ -3149,6 +3352,7 @@ version = "8.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"}, {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"}, @@ -3172,6 +3376,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -3186,6 +3391,7 @@ version = "1.2.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61"}, {file = "python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6"}, @@ -3200,6 +3406,7 @@ version = "3.5.0" description = "JOSE implementation in Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "python_jose-3.5.0-py2.py3-none-any.whl", hash = "sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771"}, {file = "python_jose-3.5.0.tar.gz", hash = "sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b"}, @@ -3222,6 +3429,7 @@ version = "0.0.21" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "python_multipart-0.0.21-py3-none-any.whl", hash = "sha256:cf7a6713e01c87aa35387f4774e812c4361150938d20d232800f75ffcf266090"}, {file = "python_multipart-0.0.21.tar.gz", hash = "sha256:7137ebd4d3bbf70ea1622998f902b97a29434a9e8dc40eb203bbcf7c2a2cba92"}, @@ -3233,6 +3441,7 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -3244,6 +3453,8 @@ version = "2020.1" description = "The Olson timezone database for Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, @@ -3255,6 +3466,8 @@ version = "311" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["dev"] +markers = "sys_platform == \"win32\"" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -3284,6 +3497,8 @@ version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, @@ -3295,6 +3510,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -3377,6 +3593,7 @@ version = "1.1" description = "A custom YAML tag for referencing environment variables in YAML files." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, @@ -3391,6 +3608,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -3412,6 +3630,7 @@ version = "13.9.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, @@ -3431,6 +3650,7 @@ version = "4.9.1" description = "Pure-Python RSA implementation" optional = false python-versions = "<4,>=3.6" +groups = ["main"] files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, @@ -3445,6 +3665,7 @@ version = "0.8.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"}, {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"}, @@ -3472,6 +3693,7 @@ version = "2025.12.0" description = "Convenient Filesystem interface over S3" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "s3fs-2025.12.0-py3-none-any.whl", hash = "sha256:89d51e0744256baad7ae5410304a368ca195affd93a07795bc8ba9c00c9effbb"}, {file = "s3fs-2025.12.0.tar.gz", hash = "sha256:8612885105ce14d609c5b807553f9f9956b45541576a17ff337d9435ed3eb01f"}, @@ -3488,6 +3710,7 @@ version = "0.13.1" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724"}, {file = "s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf"}, @@ -3505,19 +3728,20 @@ version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] -core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -3525,6 +3749,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -3536,6 +3761,7 @@ version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -3547,6 +3773,7 @@ version = "2.0.45" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "sqlalchemy-2.0.45-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c64772786d9eee72d4d3784c28f0a636af5b0a29f3fe26ff11f55efe90c0bd85"}, {file = "sqlalchemy-2.0.45-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7ae64ebf7657395824a19bca98ab10eb9a3ecb026bf09524014f1bb81cb598d4"}, @@ -3637,6 +3864,7 @@ version = "0.46.2" description = "The little ASGI library that shines." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35"}, {file = "starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5"}, @@ -3654,6 +3882,7 @@ version = "1.7.3" description = "Strict, typed YAML parser" optional = false python-versions = ">=3.7.0" +groups = ["main"] files = [ {file = "strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7"}, {file = "strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407"}, @@ -3668,6 +3897,7 @@ version = "9.1.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, @@ -3683,6 +3913,7 @@ version = "4.13.3" description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false python-versions = ">=3.9.2" +groups = ["dev"] files = [ {file = "testcontainers-4.13.3-py3-none-any.whl", hash = "sha256:063278c4805ffa6dd85e56648a9da3036939e6c0ac1001e851c9276b19b05970"}, {file = "testcontainers-4.13.3.tar.gz", hash = "sha256:9d82a7052c9a53c58b69e1dc31da8e7a715e8b3ec1c4df5027561b47e2efe646"}, @@ -3701,7 +3932,7 @@ aws = ["boto3", "httpx"] azurite = ["azure-storage-blob (>=12.19,<13.0)"] chroma = ["chromadb-client (>=1.0.0,<2.0.0)"] cosmosdb = ["azure-cosmos"] -db2 = ["ibm_db_sa", "sqlalchemy"] +db2 = ["ibm_db_sa ; platform_machine != \"aarch64\" and platform_machine != \"arm64\"", "sqlalchemy"] generic = ["httpx", "redis"] google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] influxdb = ["influxdb", "influxdb-client"] @@ -3711,12 +3942,12 @@ localstack = ["boto3"] mailpit = ["cryptography"] minio = ["minio"] mongodb = ["pymongo"] -mssql = ["pymssql (>=2.3.9)", "sqlalchemy"] +mssql = ["pymssql (>=2.3.9) ; platform_machine != \"arm64\" or python_version >= \"3.10\"", "sqlalchemy"] mysql = ["pymysql[rsa]", "sqlalchemy"] nats = ["nats-py"] neo4j = ["neo4j"] -openfga = ["openfga-sdk"] -opensearch = ["opensearch-py"] +openfga = ["openfga-sdk ; python_version >= \"3.10\""] +opensearch = ["opensearch-py ; python_version < \"4.0\""] oracle = ["oracledb (>=3.4.1)", "sqlalchemy"] oracle-free = ["oracledb (>=3.4.1)", "sqlalchemy"] qdrant = ["qdrant-client"] @@ -3736,6 +3967,8 @@ version = "2.3.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, @@ -3787,6 +4020,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -3808,6 +4042,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -3819,6 +4054,7 @@ version = "2025.3" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["dev"] files = [ {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, @@ -3830,16 +4066,17 @@ version = "2.6.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd"}, {file = "urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797"}, ] [package.extras] -brotli = ["brotli (>=1.2.0)", "brotlicffi (>=1.2.0.0)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "uvicorn" @@ -3847,6 +4084,7 @@ version = "0.32.1" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "uvicorn-0.32.1-py3-none-any.whl", hash = "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e"}, {file = "uvicorn-0.32.1.tar.gz", hash = "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175"}, @@ -3858,7 +4096,29 @@ h11 = ">=0.8" typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "virtualenv" +version = "20.35.4" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b"}, + {file = "virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" +typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "watchdog" @@ -3866,6 +4126,7 @@ version = "6.0.0" description = "Filesystem events monitoring" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, @@ -3908,6 +4169,7 @@ version = "1.0.0" description = "'Turn functions and methods into fully controllable objects'" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "wirerope-1.0.0-py2.py3-none-any.whl", hash = "sha256:59346555c7b5dbd1c683a4e123f8bed30ca99df646f6867ea6439ceabf43c2f6"}, {file = "wirerope-1.0.0.tar.gz", hash = "sha256:7da8bb6feeff9dd939bd7141ef0dc392674e43ba662e20909d6729db81a7c8d0"}, @@ -3918,7 +4180,7 @@ six = ">=1.11.0" [package.extras] doc = ["sphinx"] -test = ["pytest (>=4.6.7)", "pytest-checkdocs (>=1.2.5)", "pytest-checkdocs (>=2.9.0)", "pytest-cov (>=2.6.1)"] +test = ["pytest (>=4.6.7)", "pytest-checkdocs (>=1.2.5) ; python_version < \"3\"", "pytest-checkdocs (>=2.9.0) ; python_version >= \"3\"", "pytest-cov (>=2.6.1)"] [[package]] name = "wrapt" @@ -3926,6 +4188,7 @@ version = "1.17.3" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04"}, {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2"}, @@ -4016,6 +4279,7 @@ version = "3.2.9" description = "A Python module for creating Excel XLSX files." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3"}, {file = "xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c"}, @@ -4027,6 +4291,7 @@ version = "1.22.0" description = "Yet another URL library" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "yarl-1.22.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c7bd6683587567e5a49ee6e336e0612bec8329be1b7d4c8af5687dcdeb67ee1e"}, {file = "yarl-1.22.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5cdac20da754f3a723cceea5b3448e1a2074866406adeb4ef35b469d089adb8f"}, @@ -4166,6 +4431,6 @@ multidict = ">=4.0" propcache = ">=0.2.1" [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "b9bf207c94851e84665441e9d985573867aa8f81a3eaaa0392f84f69ed5d416a" +content-hash = "2f72479ac0a544801741db44a14d933df18bf32f348c1003b85e16911f5828ec" diff --git a/pyproject.toml b/pyproject.toml index 56fe20e37..5d1f5e26f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ mkdocstrings = "^0.30.0" mkdocstrings-python = "^1.16.12" griffe-pydantic = "^1.1.6" ruff = "^0.8.0" +pre-commit = "^4.0.0" [build-system] requires = ["poetry-core"] @@ -162,4 +163,4 @@ extend-immutable-calls = [ "fastapi.Form", "fastapi.File", "fastapi.Security", -] \ No newline at end of file +] diff --git a/readme-pypi.md b/readme-pypi.md index 5a0faadaf..ca71dd92a 100644 --- a/readme-pypi.md +++ b/readme-pypi.md @@ -6,11 +6,11 @@

Main Repository: Edwardvaneechoud/Flowfile
- Documentation: - Website - - Core - - Worker - - Frontend - + Documentation: + Website - + Core - + Worker - + Frontend - Technical Architecture

@@ -214,4 +214,4 @@ For the complete visual ETL experience, you have additional options: ## 📋 Development Roadmap -See the [main repository](https://github.com/Edwardvaneechoud/Flowfile#-todo) for the latest development roadmap and TODO list. \ No newline at end of file +See the [main repository](https://github.com/Edwardvaneechoud/Flowfile#-todo) for the latest development roadmap and TODO list. diff --git a/tools/migrate/README.md b/tools/migrate/README.md index 25f015f90..dd2c00b6d 100644 --- a/tools/migrate/README.md +++ b/tools/migrate/README.md @@ -53,4 +53,4 @@ connections: - [1, 2] node_starts: - 1 -``` \ No newline at end of file +``` diff --git a/tools/migrate/migrate.py b/tools/migrate/migrate.py index b20ad3d85..ba1a61269 100644 --- a/tools/migrate/migrate.py +++ b/tools/migrate/migrate.py @@ -99,7 +99,7 @@ def convert_to_dict(obj: Any, _seen: set = None) -> Any: return None # Handle primitives - if isinstance(obj, (str, int, float, bool)): + if isinstance(obj, str | int | float | bool): return obj # Cycle detection @@ -137,7 +137,7 @@ def convert_to_dict(obj: Any, _seen: set = None) -> Any: return {k: convert_to_dict(v, _seen) for k, v in obj.items()} # Handle lists and tuples - convert both to lists for clean YAML - if isinstance(obj, (list, tuple)): + if isinstance(obj, list | tuple): return [convert_to_dict(item, _seen) for item in obj] # Handle sets