fix: ensure compatibility with pandas>=2.0 by using SQLAlchemy Connection in read_sql

.pre-commit-config.yaml

@@ -15,11 +15,11 @@ repos:
     hooks:
       - id: formatting
         name: "isort and black"
-        exclude: ^(tests/conftest.py)
+        exclude: ^tests/conftest\.py$
         entry: bash -c '(isort "$@" && black "$@") || true; git add -u' --
         language: system
         types: [python]
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
@@ -28,16 +28,16 @@ repos:
         entry: pylint
         language: system
         types: [python]
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
       - id: pydocstyle
         name: pydocstyle
-        exclude: ^(tests/.*|demo/*)
+        exclude: ^(tests/.*|demo/.*)$
         language: system
         entry: pydocstyle
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
@@ -46,7 +46,7 @@ repos:
         entry: flake8
         language: system
         types: [python]
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
@@ -56,24 +56,24 @@ repos:
         entry: yamllint
         language: python
         types: [file, yaml]
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
       - id: pytest-check
         name: pytest-check-src
         entry: python -m pytest tests --cov=dynamicio --cov-report=html --cov-report=xml -v
-        exclude: ^(.github|.circleci|docs|.flake8|.gitlint|.pylintrc|.docs.Dockerfile|README.md|Makefile|setup.py)
+        exclude: ^(\.github|\.circleci|docs|\.flake8|\.gitlint|\.pylintrc|\.docs\.Dockerfile|README\.md|Makefile|setup\.py)$
         language: system
         pass_filenames: false
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: local
     hooks:
       - id: pytest-check
         name: pytest-check-demo
         entry: python -m pytest demo/tests
-        exclude: ^(.github|.circleci|docs|.flake8|.gitlint|.pylintrc|.docs.Dockerfile|README.md|Makefile|setup.py)
+        exclude: ^(\.github|\.circleci|docs|\.flake8|\.gitlint|\.pylintrc|\.docs\.Dockerfile|README\.md|Makefile|setup\.py)$
         language: system
         pass_filenames: false
-        stages: [commit]
+        stages: [pre-commit]

.pylintrc

@@ -383,10 +383,8 @@ preferred-modules=
 
 
 [EXCEPTIONS]
-
-# Exceptions that will emit a warning when caught.
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception
 
 
 [REFACTORING]

Makefile

@@ -37,6 +37,7 @@ dev-env-setup: install-poetry install-binary-packages-Darwin
 	@pyenv exec poetry self add poetry-pre-commit-plugin
 	@pyenv exec poetry run pre-commit install
 	@pyenv exec poetry run pre-commit install --hook-type commit-msg
+	@pyenv exec poetry run pre-commit migrate-config
 
 # Checks for local development
 check-linting:

dynamicio/__init__.py

@@ -1,4 +1,5 @@
 """A package for wrapping your I/O operations."""
+
 # pylint: disable=abstract-method
 import os
 from contextlib import suppress

dynamicio/__main__.py

@@ -1,4 +1,6 @@
 """Invokes dynamicio cli."""
+
+# Application Imports
 from dynamicio.cli import run
 
 run()

dynamicio/cli.py

@@ -1,4 +1,5 @@
 """Implements the dynamicio Command Line Interface (CLI)."""
+
 import argparse
 import glob
 import os
@@ -8,6 +9,7 @@
 import pandas as pd  # type: ignore
 import yaml
 
+# Application Imports
 from dynamicio.errors import InvalidDatasetTypeError
 
 

dynamicio/config/io_config.py

@@ -52,6 +52,7 @@
             }
         }
 """
+
 __all__ = ["IOConfig", "SafeDynamicResourceLoader", "SafeDynamicSchemaLoader"]
 
 import re
@@ -62,6 +63,7 @@
 import yaml
 from magic_logger import logger
 
+# Application Imports
 from dynamicio.config.pydantic import BindingsYaml, IOEnvironment
 
 

dynamicio/config/pydantic/__init__.py

@@ -1,14 +1,15 @@
 """Pydantic config models."""
 
+# Application Imports
 from dynamicio.config.pydantic.config import BindingsYaml
 from dynamicio.config.pydantic.io_resources import (
+    AthenaDataEnvironment,
     IOEnvironment,
     KafkaDataEnvironment,
     LocalBatchDataEnvironment,
     LocalDataEnvironment,
     PostgresDataEnvironment,
     S3DataEnvironment,
     S3PathPrefixEnvironment,
-    AthenaDataEnvironment
 )
 from dynamicio.config.pydantic.table_schema import DataframeSchema, SchemaColumn

dynamicio/config/pydantic/config.py

@@ -1,10 +1,12 @@
+"""Pydantic schema for YAML files."""
+
 # pylint: disable=no-member, no-self-argument, unused-argument
-"""Pydantic schema for YAML files"""
 
 from typing import Mapping, MutableMapping
 
 import pydantic
 
+# Application Imports
 import dynamicio.config.pydantic.io_resources as env_spec
 
 
@@ -21,14 +23,15 @@ def _validate_bindings(cls, value: Mapping):
         if not isinstance(value, Mapping):
             raise ValueError(f"Bindings must be a mapping. (got {value!r} instead).")
         # Tell each binding its name
-        for (name, sub_config) in value.items():
+        for name, sub_config in value.items():
             if not isinstance(sub_config, MutableMapping):
                 raise ValueError(f"Each element for the name binding must be a dict. (got {sub_config!r} instead)")
             sub_config["__binding_name__"] = name
         return value
 
     def update_config_refs(self) -> "BindingsYaml":
-        """Updates dynamic parts of the config:
+        """Updates dynamic parts of the config.
+
         - Configure _parent for all `IOEnvironment`s
         - Replace all IOSchemaRef with actual schema objects
         """

dynamicio/config/pydantic/io_resources.py

@@ -1,7 +1,7 @@
-# pylint: disable=no-member, no-self-argument, unused-argument
-
 """This module contains pylint models for physical data sources (places the bytes are being read from)."""
 
+# pylint: disable=no-member, no-self-argument, unused-argument, broad-exception-raised
+
 import enum
 import posixpath
 from typing import Mapping, Optional, Union

dynamicio/config/pydantic/table_schema.py

@@ -1,6 +1,7 @@
+"""This module defines Config schema for data source (pandas dataframe)."""
+
 # pylint: disable=no-member, no-self-argument, unused-argument
 
-"""This module defines Config schema for data source (pandas dataframe)"""
 
 import enum
 from typing import Mapping, Sequence
@@ -51,11 +52,11 @@ def is_valid_pandas_type(cls, info):
 
     @pydantic.validator("validations", pre=True)
     def remap_validations(cls, info):
-        """Remap the yaml structure of {validation_type: <params>} to a list with validation_type as a key"""
+        """Remap the yaml structure of {validation_type: <params>} to a list with validation_type as a key."""
         if not isinstance(info, dict):
             raise ValueError(f"{info!r} should be a dict")
         out = []
-        for (key, params) in info.items():
+        for key, params in info.items():
             new_el = params.copy()
             new_el.update({"name": key})
             out.append(new_el)
@@ -79,7 +80,7 @@ class DataframeSchema(pydantic.BaseModel):
 
     @pydantic.validator("columns", pre=True)
     def supply_column_names(cls, info):
-        """Tell each column its name (the key it is listed under)"""
+        """Tell each column its name (the key it is listed under)."""
         if not isinstance(info, Mapping):
             raise ValueError(f"{info!r} shoudl be a dict.")
 

dynamicio/core.py

@@ -1,5 +1,6 @@
 """Implements the DynamicDataIO class which provides functionality for data: loading; sinking, and; schema validation."""
-# pylint: disable=no-member
+
+# pylint: disable=too-many-positional-arguments, disable=no-member
 __all__ = ["DynamicDataIO", "SCHEMA_FROM_FILE", "CASTING_WARNING_MSG"]
 
 import asyncio
@@ -100,7 +101,7 @@ def _schema_from_obj(target) -> DataframeSchema:
             - CAN have `schema_validations` and `schema_metrics` attributes
         """
         col_info = {}
-        for (col_name, dtype) in target.schema.items():
+        for col_name, dtype in target.schema.items():
             col_validations = {}
             col_metrics = []
             try:

dynamicio/errors.py

@@ -1,4 +1,5 @@
 """Hosts exception implementations for different errors."""
+
 # pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring, super-init-not-called
 __all__ = [
     "DynamicIOError",

dynamicio/metrics.py

@@ -1,4 +1,5 @@
 """A module responsible for metrics generation and logging."""
+
 # pylint: disable=missing-function-docstring,missing-class-docstring
 import json
 import logging

dynamicio/mixins/utils.py

@@ -1,4 +1,5 @@
 """Mixin utility functions."""
+
 # pylint: disable=no-member, protected-access, too-few-public-methods
 
 import inspect

dynamicio/mixins/with_kafka.py

@@ -1,5 +1,6 @@
 """This module provides mixins that are providing Kafka I/O support."""
-# pylint: disable=no-member, protected-access, too-few-public-methods
+
+# pylint: disable=no-member, protected-access, too-few-public-methods, broad-exception-raised
 
 from typing import Any, Callable, Mapping, MutableMapping, Optional