adding comment to init.py to try to solve CI CD issues

AGENTS.md

@@ -404,7 +404,7 @@ ca-biositing/
 │       │   │       │   ├── transform/       # Transform tasks
 │       │   │       │   └── load/            # Load tasks
 │       │   │       ├── flows/               # Prefect flows
-│       │   │       │   ├── primary_product.py
+│       │   │       │   ├── primary_ag_product.py
 │       │   │       │   ├── analysis_type.py
 │       │   │       │   └── ...
 │       │   │       └── utils/               # Utilities

alembic/versions/15467b7f2f3d_test_after_revert.py

@@ -0,0 +1,32 @@
+"""test after revert
+
+Revision ID: 15467b7f2f3d
+Revises: e84690898528
+Create Date: 2025-12-25 05:02:50.212384
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '15467b7f2f3d'
+down_revision: Union[str, Sequence[str], None] = 'e84690898528'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py

@@ -0,0 +1,56 @@
+"""changing primary crop to primary_ag_product
+
+Revision ID: e84690898528
+Revises: fc422d3d9ea6
+Create Date: 2025-12-23 16:04:40.644736
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'e84690898528'
+down_revision: Union[str, Sequence[str], None] = 'fc422d3d9ea6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('primary_ag_product',
+    sa.Column('note', sa.Text(), nullable=True),
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=True),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('uri', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.drop_table('primary_crop')
+    op.add_column('resource', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True))
+    op.drop_column('resource', 'primary_crop_id')
+    op.add_column('resource_usda_commodity_map', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True))
+    op.drop_column('resource_usda_commodity_map', 'primary_crop_id')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('resource_usda_commodity_map', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.drop_column('resource_usda_commodity_map', 'primary_ag_product_id')
+    op.add_column('resource', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.drop_column('resource', 'primary_ag_product_id')
+    op.create_table('primary_crop',
+    sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True),
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True),
+    sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True),
+    sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('primary_crop_pkey'))
+    )
+    op.drop_table('primary_ag_product')
+    # ### end Alembic commands ###

docs/README.md

@@ -1 +1 @@
-../README.md
+../README.md

docs/datamodels/README.md

@@ -1 +1 @@
-../../src/ca_biositing/datamodels/README.md
+../../src/ca_biositing/datamodels/README.md

docs/deployment/README.md

@@ -1 +1 @@
-../../deployment/README.md
+../../deployment/README.md

docs/pipeline/README.md

@@ -1 +1 @@
-../../src/ca_biositing/pipeline/README.md
+../../src/ca_biositing/pipeline/README.md

docs/resources/README.md

@@ -1 +1 @@
-../../resources/README.md
+../../resources/README.md

docs/webservice/README.md

@@ -1 +1 @@
-../../src/ca_biositing/webservice/README.md
+../../src/ca_biositing/webservice/README.md

pixi.toml

@@ -242,6 +242,7 @@ frontend-build = { cmd = "npm run build", cwd = "frontend", description = "Build
 # Kernel Dependencies
 # This is to enable Jupyter Notebooks with IPython kernel
 [feature.kernel.dependencies]
+jupyter = ">=1.0.0,<2"
 ipykernel = ">=7.1.0,<8"
 ipython = ">=9.0.0,<10"
 

resources/prefect/run_prefect_flow.py

@@ -1,24 +1,32 @@
 import sys
-from prefect import flow
-from ca_biositing.pipeline.flows.primary_product import primary_product_flow
-from ca_biositing.pipeline.flows.analysis_type import analysis_type_flow
+import traceback
+from prefect import flow, get_run_logger
+from prefect.utilities.importtools import import_object
 
-# A dictionary mapping flow names to their function objects
+# A dictionary mapping flow names to their import paths
 AVAILABLE_FLOWS = {
-    "primary_product": primary_product_flow,
-    "analysis_type": analysis_type_flow,
+    "primary_ag_product": "ca_biositing.pipeline.flows.primary_ag_product.primary_ag_product_flow",
+    "analysis_type": "ca_biositing.pipeline.flows.analysis_type.analysis_type_flow",
 }
 
 @flow(name="Master ETL Flow", log_prints=True)
 def master_flow():
     """
     A master flow to orchestrate all ETL pipelines.
+    This flow dynamically imports and runs sub-flows, allowing it to continue
+    even if some sub-flows fail to import or run.
     """
-    print("Running master ETL flow...")
-    for flow_name, flow_func in AVAILABLE_FLOWS.items():
-        print(f"--- Running sub-flow: {flow_name} ---")
-        flow_func()
-    print("Master ETL flow completed.")
+    logger = get_run_logger()
+    logger.info("Running master ETL flow...")
+    for flow_name, flow_path in AVAILABLE_FLOWS.items():
+        try:
+            logger.info(f"--- Running sub-flow: {flow_name} ---")
+            flow_func = import_object(flow_path)
+            flow_func()
+        except Exception as e:
+            logger.error(f"Flow '{flow_name}' failed with error: {e}")
+            logger.error(traceback.format_exc())
+    logger.info("Master ETL flow completed.")
 
 if __name__ == "__main__":
     # This script is a placeholder for running flows directly.

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml

@@ -84,7 +84,7 @@ classes:
     description: Mapping between resources/crops and USDA commodities.
     slots:
       - resource_id
-      - primary_crop_id
+      - primary_ag_product_id
       - usda_commodity_id
       - match_tier
       - note
@@ -354,19 +354,19 @@ slots:
 
   main_crop:
     range: integer
-    description: Reference to PrimaryCrop.
+    description: Reference to PrimaryAgProduct.
 
   secondary_crop:
     range: integer
-    description: Reference to PrimaryCrop.
+    description: Reference to PrimaryAgProduct.
 
   tertiary_crop:
     range: integer
-    description: Reference to PrimaryCrop.
+    description: Reference to PrimaryAgProduct.
 
   quaternary_crop:
     range: integer
-    description: Reference to PrimaryCrop.
+    description: Reference to PrimaryAgProduct.
 
   confidence:
     range: integer

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml

@@ -0,0 +1,23 @@
+id: https://w3id.org/ca_biositing/primary_ag_product
+name: primary_ag_product
+title: Primary Agricultural Product
+description: Core definitions for primary agricultural products.
+license: https://creativecommons.org/publicdomain/zero/1.0/
+
+prefixes:
+  linkml: https://w3id.org/linkml/
+  ca_biositing: https://w3id.org/ca_biositing/
+
+default_prefix: ca_biositing
+default_range: string
+
+imports:
+  - linkml:types
+  - core
+
+classes:
+  PrimaryAgProduct:
+    is_a: LookupBase
+    description: Primary agricultural product definition.
+    slots:
+      - note

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml

@@ -0,0 +1,35 @@
+id: https://w3id.org/ca_biositing/resource_availability
+name: resource_availability
+title: Resource Availability
+description: Availability of a resource in a location.
+license: https://creativecommons.org/publicdomain/zero/1.0/
+
+prefixes:
+  linkml: https://w3id.org/linkml/
+  ca_biositing: https://w3id.org/ca_biositing/
+
+default_prefix: ca_biositing
+default_range: string
+
+imports:
+  - linkml:types
+  - core
+  - places
+  - resource_information
+
+classes:
+  ResourceAvailability:
+    is_a: BaseEntity
+    description: Availability of a resource in a location.
+    slots:
+      - resource_id
+      - geoid
+      - from_month
+      - to_month
+      - year_round
+      - note
+
+slots:
+  resource_id:
+    range: Resource
+    description: Reference to Resource.

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml

@@ -0,0 +1,21 @@
+id: https://w3id.org/ca_biositing/resource_class
+name: resource_class
+title: Resource Class
+description: Classification of resources.
+license: https://creativecommons.org/publicdomain/zero/1.0/
+
+prefixes:
+  linkml: https://w3id.org/linkml/
+  ca_biositing: https://w3id.org/ca_biositing/
+
+default_prefix: ca_biositing
+default_range: string
+
+imports:
+  - linkml:types
+  - core
+
+classes:
+  ResourceClass:
+    is_a: LookupBase
+    description: Classification of resources.

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml

@@ -0,0 +1,47 @@
+id: https://w3id.org/ca_biositing/resource_counterfactual
+name: resource_counterfactual
+title: Resource Counterfactual
+description: Counterfactual uses of a resource.
+license: https://creativecommons.org/publicdomain/zero/1.0/
+
+prefixes:
+  linkml: https://w3id.org/linkml/
+  ca_biositing: https://w3id.org/ca_biositing/
+
+default_prefix: ca_biositing
+default_range: string
+
+imports:
+  - linkml:types
+  - core
+  - places
+  - resource_information
+  - data_sources_metadata
+
+classes:
+  ResourceCounterfactual:
+    is_a: BaseEntity
+    description: Counterfactual uses of a resource.
+    slots:
+      - geoid
+      - resource_id
+      - counterfactual_description
+      - animal_bedding_percent
+      - animal_bedding_source_id
+      - animal_feed_percent
+      - animal_feed_source_id
+      - bioelectricty_percent
+      - bioelectricty_source_id
+      - burn_percent
+      - burn_source_id
+      - compost_percent
+      - compost_source_id
+      - landfill_percent
+      - landfill_source_id
+      - counterfactual_date
+      - note
+
+slots:
+  resource_id:
+    range: Resource
+    description: Reference to Resource.

src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml

@@ -22,7 +22,7 @@ classes:
     description: Biomass resource definition.
     slots:
       - name
-      - primary_crop_id
+      - primary_ag_product_id
       - resource_class_id
       - resource_subclass_id
       - note
@@ -36,9 +36,9 @@ classes:
     is_a: LookupBase
     description: Sub-classification of resources.
 
-  PrimaryCrop:
+  PrimaryAgProduct:
     is_a: LookupBase
-    description: Primary crop definition.
+    description: Primary agricultural product definition.
     slots:
       - note
 
@@ -89,9 +89,9 @@ classes:
 slots:
   #Resource slots
 
-  primary_crop_id:
+  primary_ag_product_id:
     range: integer
-    description: Reference to PrimaryCrop.
+    description: Reference to PrimaryAgProduct.
 
   resource_class_id:
     range: integer