feat: ETL Pipeline Notebooks and Fix LinkML SQLAlchemy generation

AGENTS.md

@@ -404,7 +404,7 @@ ca-biositing/
 │       │   │       │   ├── transform/       # Transform tasks
 │       │   │       │   └── load/            # Load tasks
 │       │   │       ├── flows/               # Prefect flows
-│       │   │       │   ├── primary_product.py
+│       │   │       │   ├── primary_ag_product.py
 │       │   │       │   ├── analysis_type.py
 │       │   │       │   └── ...
 │       │   │       └── utils/               # Utilities

alembic/env.py

@@ -24,16 +24,12 @@
 # from ca_biositing.datamodels.sample_preprocessing import *
 # from ca_biositing.datamodels.specific_aalysis_results import *
 # from ca_biositing.datamodels.user import *
+from ca_biositing.datamodels.schemas.generated import ca_biositing
 from ca_biositing.datamodels.schemas.generated.ca_biositing import *
-from sqlmodel import SQLModel
-import importlib.util
-from pathlib import Path
-
-# --- Import generated models and their metadata ---
-# from ca_biositing.datamodels.schemas.generated.census_survey import metadata as census_metadata
-# from ca_biositing.datamodels.schemas.generated.geography import metadata as geography_metadata
+# from ca_biositing.datamodels.database import Base
 
-# --- Alembic Config object, provides access to alembic.ini values ---
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
 config = context.config
 
 # Override sqlalchemy.url in alembic.ini with value from .env
@@ -43,16 +39,15 @@
 else:
     raise RuntimeError("DATABASE_URL not found in .env file. Alembic cannot run migrations.")
 
-# --- Configure logging (from alembic.ini logging section) ---
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
-# --- Metadata from your models for autogenerate ---
-# Combine metadata from all models
-# See: https://alembic.sqlalchemy.org/en/latest/autogenerate.html#affecting-the-autogenerate-process
-
-# Start with the default SQLModel metadata
-# target_metadata = SQLModel.metadata
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
 # Manually merge metadata from generated modules

alembic/versions/15467b7f2f3d_test_after_revert.py

@@ -0,0 +1,32 @@
+"""test after revert
+
+Revision ID: 15467b7f2f3d
+Revises: e84690898528
+Create Date: 2025-12-25 05:02:50.212384
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '15467b7f2f3d'
+down_revision: Union[str, Sequence[str], None] = 'e84690898528'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/1ace3fc9c593_initial_migration.py

@@ -0,0 +1,32 @@
+"""Initial migration
+
+Revision ID: 1ace3fc9c593
+Revises: 62ef89b0050a
+Create Date: 2026-01-07 18:39:13.836533
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '1ace3fc9c593'
+down_revision: Union[str, Sequence[str], None] = '62ef89b0050a'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/2b18e6e5e72d_trying_with_fixing_the_base.py

@@ -0,0 +1,32 @@
+"""trying with fixing the base
+
+Revision ID: 2b18e6e5e72d
+Revises: 1ace3fc9c593
+Create Date: 2026-01-07 20:55:34.214689
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '2b18e6e5e72d'
+down_revision: Union[str, Sequence[str], None] = '1ace3fc9c593'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/2b41e390fc5d_fixed_alembic_environment_to_import_.py

@@ -0,0 +1,32 @@
+"""Fixed alembic environment to import generated schemas again
+
+Revision ID: 2b41e390fc5d
+Revises: a8a3ea954e88
+Create Date: 2026-01-07 18:18:18.527886
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '2b41e390fc5d'
+down_revision: Union[str, Sequence[str], None] = 'a8a3ea954e88'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/62ef89b0050a_fixed_alembic_gen_sqla_so_all_inherit_.py

@@ -0,0 +1,32 @@
+"""Fixed alembic gen_sqla so all inherit from same base
+
+Revision ID: 62ef89b0050a
+Revises: 2b41e390fc5d
+Create Date: 2026-01-07 18:33:17.030868
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '62ef89b0050a'
+down_revision: Union[str, Sequence[str], None] = '2b41e390fc5d'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###