From 5574630951e0d6a65d0bbdadda22308f444d122f Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Mon, 22 Dec 2025 14:55:55 -0700 Subject: [PATCH 1/7] adding comment to init.py to try to solve CI CD issues --- .../datamodels/ca_biositing/datamodels/schemas/__init__.py | 1 + .../ca_biositing/datamodels/schemas/generated/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py index dbb70e85..6279548e 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py @@ -1 +1,2 @@ # This file makes this a Python package. +# Schemas package. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py index dbb70e85..73de05be 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py @@ -1 +1,2 @@ # This file makes this a Python package. +# Generated schemas. From e0947fc73d0de22736c4a7c117719f7a7df9a42c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 23 Dec 2025 10:12:53 -0700 Subject: [PATCH 2/7] fixed primary_ag_product etl pipeline, modified master flow script and docs --- AGENTS.md | 2 +- ...28_changing_primary_crop_to_primary_ag_.py | 56 +++++++++++++++++++ docs/README.md | 2 +- docs/datamodels/README.md | 2 +- docs/deployment/README.md | 2 +- docs/pipeline/README.md | 2 +- docs/resources/README.md | 2 +- docs/webservice/README.md | 2 +- resources/prefect/run_prefect_flow.py | 30 ++++++---- .../linkml/modules/external_data.yaml | 10 ++-- .../linkml/modules/resource_information.yaml | 10 ++-- .../schemas/generated/aim1_records.py | 12 ++-- .../schemas/generated/aim2_records.py | 12 ++-- .../schemas/generated/ca_biositing.py | 16 +++--- .../generated/data_sources_metadata.py | 12 ++-- .../schemas/generated/experiment_equipment.py | 12 ++-- .../schemas/generated/external_data.py | 16 +++--- .../schemas/generated/field_sampling.py | 12 ++-- .../schemas/generated/general_analysis.py | 12 ++-- .../schemas/generated/infrastructure.py | 12 ++-- .../generated/methods_parameters_units.py | 12 ++-- .../schemas/generated/resource_information.py | 12 ++-- .../schemas/generated/sample_preparation.py | 12 ++-- src/ca_biositing/pipeline/AGENTS.md | 24 +++++--- src/ca_biositing/pipeline/README.md | 35 ++++++------ ...imary_product.py => primary_ag_product.py} | 24 ++++---- .../transform/products/primary_ag_product.py | 41 ++++++++++++++ .../etl/transform/products/primary_product.py | 38 ------------- .../pipeline/flows/primary_ag_product.py | 26 +++++++++ .../pipeline/flows/primary_product.py | 26 --------- 30 files changed, 280 insertions(+), 206 deletions(-) create mode 100644 alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py rename src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/{primary_product.py => primary_ag_product.py} (61%) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py diff --git a/AGENTS.md b/AGENTS.md index 45e1e109..f65c5373 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -404,7 +404,7 @@ ca-biositing/ │ │ │ │ ├── transform/ # Transform tasks │ │ │ │ └── load/ # Load tasks │ │ │ ├── flows/ # Prefect flows -│ │ │ │ ├── primary_product.py +│ │ │ │ ├── primary_ag_product.py │ │ │ │ ├── analysis_type.py │ │ │ │ └── ... │ │ │ └── utils/ # Utilities diff --git a/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py b/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py new file mode 100644 index 00000000..d7efd056 --- /dev/null +++ b/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py @@ -0,0 +1,56 @@ +"""changing primary crop to primary_ag_product + +Revision ID: e84690898528 +Revises: fc422d3d9ea6 +Create Date: 2025-12-23 16:04:40.644736 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e84690898528' +down_revision: Union[str, Sequence[str], None] = 'fc422d3d9ea6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('primary_ag_product', + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.drop_table('primary_crop') + op.add_column('resource', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True)) + op.drop_column('resource', 'primary_crop_id') + op.add_column('resource_usda_commodity_map', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True)) + op.drop_column('resource_usda_commodity_map', 'primary_crop_id') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('resource_usda_commodity_map', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column('resource_usda_commodity_map', 'primary_ag_product_id') + op.add_column('resource', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column('resource', 'primary_ag_product_id') + op.create_table('primary_crop', + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('primary_crop_pkey')) + ) + op.drop_table('primary_ag_product') + # ### end Alembic commands ### diff --git a/docs/README.md b/docs/README.md index 94389aee..32d46ee8 120000 --- a/docs/README.md +++ b/docs/README.md @@ -1 +1 @@ -../README.md +../README.md \ No newline at end of file diff --git a/docs/datamodels/README.md b/docs/datamodels/README.md index f3c7aa23..b62101ad 120000 --- a/docs/datamodels/README.md +++ b/docs/datamodels/README.md @@ -1 +1 @@ -../../src/ca_biositing/datamodels/README.md +../../src/ca_biositing/datamodels/README.md \ No newline at end of file diff --git a/docs/deployment/README.md b/docs/deployment/README.md index 5159bcfe..d5c46ad4 120000 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -1 +1 @@ -../../deployment/README.md +../../deployment/README.md \ No newline at end of file diff --git a/docs/pipeline/README.md b/docs/pipeline/README.md index 544d50b1..82a8c831 120000 --- a/docs/pipeline/README.md +++ b/docs/pipeline/README.md @@ -1 +1 @@ -../../src/ca_biositing/pipeline/README.md +../../src/ca_biositing/pipeline/README.md \ No newline at end of file diff --git a/docs/resources/README.md b/docs/resources/README.md index ea34ca59..9f21ec35 120000 --- a/docs/resources/README.md +++ b/docs/resources/README.md @@ -1 +1 @@ -../../resources/README.md +../../resources/README.md \ No newline at end of file diff --git a/docs/webservice/README.md b/docs/webservice/README.md index 24ea2883..749a635e 120000 --- a/docs/webservice/README.md +++ b/docs/webservice/README.md @@ -1 +1 @@ -../../src/ca_biositing/webservice/README.md +../../src/ca_biositing/webservice/README.md \ No newline at end of file diff --git a/resources/prefect/run_prefect_flow.py b/resources/prefect/run_prefect_flow.py index b350f382..03f5e926 100644 --- a/resources/prefect/run_prefect_flow.py +++ b/resources/prefect/run_prefect_flow.py @@ -1,24 +1,32 @@ import sys -from prefect import flow -from ca_biositing.pipeline.flows.primary_product import primary_product_flow -from ca_biositing.pipeline.flows.analysis_type import analysis_type_flow +import traceback +from prefect import flow, get_run_logger +from prefect.utilities.importtools import import_object -# A dictionary mapping flow names to their function objects +# A dictionary mapping flow names to their import paths AVAILABLE_FLOWS = { - "primary_product": primary_product_flow, - "analysis_type": analysis_type_flow, + "primary_ag_product": "ca_biositing.pipeline.flows.primary_ag_product.primary_ag_product_flow", + "analysis_type": "ca_biositing.pipeline.flows.analysis_type.analysis_type_flow", } @flow(name="Master ETL Flow", log_prints=True) def master_flow(): """ A master flow to orchestrate all ETL pipelines. + This flow dynamically imports and runs sub-flows, allowing it to continue + even if some sub-flows fail to import or run. """ - print("Running master ETL flow...") - for flow_name, flow_func in AVAILABLE_FLOWS.items(): - print(f"--- Running sub-flow: {flow_name} ---") - flow_func() - print("Master ETL flow completed.") + logger = get_run_logger() + logger.info("Running master ETL flow...") + for flow_name, flow_path in AVAILABLE_FLOWS.items(): + try: + logger.info(f"--- Running sub-flow: {flow_name} ---") + flow_func = import_object(flow_path) + flow_func() + except Exception as e: + logger.error(f"Flow '{flow_name}' failed with error: {e}") + logger.error(traceback.format_exc()) + logger.info("Master ETL flow completed.") if __name__ == "__main__": # This script is a placeholder for running flows directly. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml index 3321a2f4..679a5f4a 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml @@ -84,7 +84,7 @@ classes: description: Mapping between resources/crops and USDA commodities. slots: - resource_id - - primary_crop_id + - primary_ag_product_id - usda_commodity_id - match_tier - note @@ -354,19 +354,19 @@ slots: main_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. secondary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. tertiary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. quaternary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. confidence: range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml index f06fa29d..82237cae 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml @@ -22,7 +22,7 @@ classes: description: Biomass resource definition. slots: - name - - primary_crop_id + - primary_ag_product_id - resource_class_id - resource_subclass_id - note @@ -36,9 +36,9 @@ classes: is_a: LookupBase description: Sub-classification of resources. - PrimaryCrop: + PrimaryAgProduct: is_a: LookupBase - description: Primary crop definition. + description: Primary agricultural product definition. slots: - note @@ -89,9 +89,9 @@ classes: slots: #Resource slots - primary_crop_id: + primary_ag_product_id: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. resource_class_id: range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py index 075f0de2..85b02306 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py @@ -892,7 +892,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -905,7 +905,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -967,11 +967,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -981,7 +981,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py index 45edb52c..374bbf8b 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py @@ -918,7 +918,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -931,7 +931,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -993,11 +993,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1007,7 +1007,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py index 734d1d17..6fc7680f 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py @@ -738,7 +738,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -751,7 +751,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -813,11 +813,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -827,7 +827,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2206,7 +2206,7 @@ class ResourceUsdaCommodityMap(BaseEntity): __tablename__ = 'resource_usda_commodity_map' resource_id = Column(Integer()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) usda_commodity_id = Column(Integer()) match_tier = Column(Text()) note = Column(Text()) @@ -2218,7 +2218,7 @@ class ResourceUsdaCommodityMap(BaseEntity): def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_crop_id={self.primary_crop_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py index 9183431c..10206eab 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py @@ -383,7 +383,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -396,7 +396,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -458,11 +458,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -472,7 +472,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py index ae56a768..d27ecc16 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py @@ -1078,7 +1078,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -1091,7 +1091,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1153,11 +1153,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1167,7 +1167,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py index d854df67..deaaf7e6 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py @@ -358,7 +358,7 @@ class ResourceUsdaCommodityMap(BaseEntity): __tablename__ = 'resource_usda_commodity_map' resource_id = Column(Integer()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) usda_commodity_id = Column(Integer()) match_tier = Column(Text()) note = Column(Text()) @@ -370,7 +370,7 @@ class ResourceUsdaCommodityMap(BaseEntity): def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_crop_id={self.primary_crop_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1041,7 +1041,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -1054,7 +1054,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1116,11 +1116,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1130,7 +1130,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py index 822de073..0e5c3b47 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py @@ -512,7 +512,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -525,7 +525,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -587,11 +587,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -601,7 +601,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py index 6470faba..5e8fd492 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py @@ -501,7 +501,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -514,7 +514,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -576,11 +576,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -590,7 +590,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py index 796558e4..021a9423 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py @@ -860,7 +860,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -873,7 +873,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -935,11 +935,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -949,7 +949,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py index 6be929f8..85ddde97 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py @@ -416,7 +416,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -429,7 +429,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -491,11 +491,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -505,7 +505,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py index bc84a0af..627f9124 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py @@ -99,7 +99,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -112,7 +112,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -174,11 +174,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -188,7 +188,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py index b543fbaa..ae68592c 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py @@ -317,7 +317,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -330,7 +330,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -392,11 +392,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -406,7 +406,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/pipeline/AGENTS.md b/src/ca_biositing/pipeline/AGENTS.md index 16b5e29a..1b9fc453 100644 --- a/src/ca_biositing/pipeline/AGENTS.md +++ b/src/ca_biositing/pipeline/AGENTS.md @@ -109,11 +109,13 @@ Located in `ca_biositing/pipeline/etl/`: - Pattern: Prefect `@task` decorated functions that return DataFrames 2. **`transform/`** - Data transformation tasks - - `products/primary_product.py`: Transform primary product data + - `products/primary_ag_product.py`: Transform primary agricultural product + data - Pattern: Prefect `@task` functions that accept and return DataFrames 3. **`load/`** - Data loading tasks - - `products/primary_product.py`: Load primary products to database + - `products/primary_ag_product.py`: Load primary agricultural products to + database - `analysis/`: Analysis-related load functions - Pattern: Prefect `@task` functions that insert data via SQLModel session @@ -126,7 +128,7 @@ Located in `ca_biositing/pipeline/etl/`: Located in `ca_biositing/pipeline/flows/`: -- **`primary_product.py`**: Primary product ETL flow +- **`primary_ag_product.py`**: Primary agricultural product ETL flow - **`analysis_type.py`**: Analysis type ETL flow - Pattern: Prefect `@flow` decorated functions that orchestrate tasks @@ -141,8 +143,10 @@ Located in `ca_biositing/pipeline/utils/`: 2. **`gsheet_to_pandas.py`** - Google Sheets to DataFrame conversion - `gsheet_to_df()`: Main extraction function -3. **`run_pipeline.py`** - Master pipeline orchestrator - - Runs all ETL flows in sequence +3. **`../../resources/prefect/run_prefect_flow.py`** - Master pipeline + orchestrator + - A resilient master flow that dynamically imports and runs all ETL flows, + catching errors from individual flows. 4. **`clear_alembic.py`** - Database migration cleanup utility @@ -442,7 +446,8 @@ def test_lookup_utility(session): 4. **Create flow** in `flows/my_flow.py`: - Use `@flow` decorator - Orchestrate extract → transform → load - - Add to master orchestrator in `utils/run_pipeline.py` + - Add the flow's import path to the `AVAILABLE_FLOWS` dictionary in + `resources/prefect/run_prefect_flow.py` 5. **Write tests** in `tests/test_my_pipeline.py`: - Mock external dependencies @@ -634,10 +639,11 @@ Run flows from command line or Python: ```bash # Run specific flow -python -m ca_biositing.pipeline.flows.primary_product +python -m ca_biositing.pipeline.flows.primary_ag_product -# Run all flows (master orchestrator) -python -m ca_biositing.pipeline.utils.run_pipeline +# Run all flows via the main project's pixi task +# This executes the master flow defined in resources/prefect/run_prefect_flow.py +pixi run run-etl ``` ### Usage with Docker diff --git a/src/ca_biositing/pipeline/README.md b/src/ca_biositing/pipeline/README.md index ac9c59e1..e0514b5f 100644 --- a/src/ca_biositing/pipeline/README.md +++ b/src/ca_biositing/pipeline/README.md @@ -33,20 +33,20 @@ src/ca_biositing/pipeline/ │ │ │ └── experiments.py │ │ ├── transform/ # Data transformation tasks │ │ │ └── products/ -│ │ │ └── primary_product.py +│ │ │ └── primary_ag_product.py │ │ ├── load/ # Data loading tasks │ │ │ ├── analysis/ │ │ │ └── products/ -│ │ │ └── primary_product.py +│ │ │ └── primary_ag_product.py │ │ └── templates/ # ETL module templates │ ├── flows/ # Prefect flow definitions │ │ ├── analysis_type.py -│ │ └── primary_product.py +│ │ └── primary_ag_product.py │ └── utils/ # Utility functions │ ├── __init__.py │ ├── gsheet_to_pandas.py │ ├── lookup_utils.py -│ └── run_pipeline.py +│ └── ../../resources/prefect/run_prefect_flow.py ├── tests/ # Test suite │ ├── __init__.py │ ├── conftest.py # Pytest fixtures @@ -89,15 +89,12 @@ directory: on SQLModel changes - **Note:** Database models are now in the shared `ca-biositing-datamodels` package -- **[See: docs/ALEMBIC_WORKFLOW.md](./ALEMBIC_WORKFLOW.md)** ### 3. ETL Pipeline Development (Prefect) - **Purpose:** Running the ETL pipeline and adding new data pipelines - **Details:** Using Prefect's flow orchestration with extract, transform, and load tasks -- **[See: docs/ETL_WORKFLOW.md](./ETL_WORKFLOW.md)** -- **[See: docs/PREFECT_WORKFLOW.md](./PREFECT_WORKFLOW.md)** ### 4. Google Cloud Setup @@ -162,7 +159,7 @@ See `tests/README.md` for detailed information about the test suite. ```python from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info -from ca_biositing.pipeline.flows.primary_product import primary_product_flow +from ca_biositing.pipeline.flows.primary_ag_product import primary_ag_product_flow from ca_biositing.pipeline.utils.lookup_utils import replace_name_with_id_df # Use in your code @@ -172,10 +169,10 @@ from ca_biositing.pipeline.utils.lookup_utils import replace_name_with_id_df ### Running Prefect Flows ```python -from ca_biositing.pipeline.flows.primary_product import primary_product_flow +from ca_biositing.pipeline.flows.primary_ag_product import primary_ag_product_flow # Run the flow -primary_product_flow() +primary_ag_product_flow() ``` ### Using Utility Functions @@ -232,7 +229,8 @@ docker-compose exec app alembic upgrade head **5. Run ETL Pipeline:** ```bash -docker-compose exec app python utils/run_pipeline.py +# This executes the master flow defined in resources/prefect/run_prefect_flow.py +pixi run run-etl ``` See `docs/DOCKER_WORKFLOW.md` and `docs/ETL_WORKFLOW.md` for detailed @@ -249,17 +247,19 @@ instructions. **Transform:** Data cleaning and transformation using pandas -- `transform/products/primary_product.py`: Transform primary product data +- `transform/products/primary_ag_product.py`: Transform primary agricultural + product data **Load:** Load transformed data into PostgreSQL -- `load/products/primary_product.py`: Load primary products into database +- `load/products/primary_ag_product.py`: Load primary agricultural products into + database ### Prefect Flows Orchestrated workflows that combine ETL tasks: -- `flows/primary_product.py`: Primary product ETL flow +- `flows/primary_ag_product.py`: Primary agricultural product ETL flow - `flows/analysis_type.py`: Analysis type ETL flow ### Utility Functions @@ -271,7 +271,9 @@ Orchestrated workflows that combine ETL tasks: **`gsheet_to_pandas.py`**: Google Sheets to pandas DataFrame conversion -**`run_pipeline.py`**: Master script to run all ETL flows +**`../../resources/prefect/run_prefect_flow.py`**: A resilient master flow that +dynamically imports and runs all ETL flows, catching errors from individual +flows. ## Dependencies @@ -304,8 +306,7 @@ pixi run pre-commit run --files src/ca_biositing/pipeline/**/* 5. **Tests:** Add tests in `tests/` 6. **Run:** Execute the flow -See `etl/templates/` for template files and `docs/ETL_WORKFLOW.md` for detailed -instructions. +See `etl/templates/` for template files ### Adding Database Models diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py similarity index 61% rename from src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py rename to src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py index 57d0d6d3..0508d3a3 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py @@ -2,37 +2,37 @@ from prefect import task, get_run_logger from sqlmodel import Session, select from ca_biositing.datamodels.database import engine -from ca_biositing.datamodels.biomass import PrimaryProduct +from ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct @task -def load_products_primary_product(primary_product_df: pd.DataFrame): +def load_products_primary_ag_product(primary_ag_product_df: pd.DataFrame): """ - Loads the data from the primary products DataFrame into the database. + Loads the data from the primary ag products DataFrame into the database. - Iterates over a DataFrame and inserts each product name from the 'Primary_crop' column - into the PrimaryProduct table. + Iterates over a DataFrame and inserts each product name from the 'name' column + into the PrimaryAgProduct table. """ logger = get_run_logger() - if primary_product_df is None or primary_product_df.empty: + if primary_ag_product_df is None or primary_ag_product_df.empty: logger.info("No data to load. Skipping database insertion.") return - column_name = 'Primary_crop' - if column_name not in primary_product_df.columns: + column_name = 'name' + if column_name not in primary_ag_product_df.columns: logger.error(f"Column '{column_name}' not found in the DataFrame. Aborting load.") return - logger.info(f"Attempting to load {len(primary_product_df)} products into the database...") + logger.info(f"Attempting to load {len(primary_ag_product_df)} products into the database...") with Session(engine) as session: - statement = select(PrimaryProduct.primary_product_name) + statement = select(PrimaryAgProduct.name) existing_products = session.exec(statement).all() existing_product_names = set(existing_products) records_to_add = [] - for product_name in primary_product_df[column_name]: + for product_name in primary_ag_product_df[column_name]: if product_name not in existing_product_names: - product = PrimaryProduct(primary_product_name=product_name) + product = PrimaryAgProduct(name=product_name) records_to_add.append(product) existing_product_names.add(product_name) # Add to set to avoid re-adding in same batch diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py new file mode 100644 index 00000000..25182c62 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py @@ -0,0 +1,41 @@ +from typing import Optional, Dict +import pandas as pd +from prefect import task, get_run_logger + +EXTRACT_SOURCES = ["basic_sample_info"] + +@task +def transform_products_primary_ag_product(data_sources: Dict[str, pd.DataFrame]) -> Optional[pd.DataFrame]: + """ + Transforms the raw data to extract unique primary agricultural products. + + This function is purely for transformation (the 'T' in ETL). It takes the + raw DataFrame and performs the specific transformations needed for the + 'primary_ag_product' table. + + Args: + data_sources: A dictionary of DataFrames from the extraction step. + + Returns: + A transformed DataFrame with a single 'name' column containing + unique product names, or None if an error occurs. + """ + logger = get_run_logger() + logger.info("Transforming raw data for primary ag products...") + + raw_df = data_sources["basic_sample_info"] + + #Step 1: Convert column names to lowercase for consistency + raw_df.columns = [col.lower() for col in raw_df.columns] + + # Step 2: Check if the required column exists + if 'primary_ag_product' not in raw_df.columns: + logger.error("'primary_ag_product' column not found in the raw data.") + return None + + # Step 3: Get unique product names and create the final DataFrame + primary_ag_product= raw_df['primary_ag_product'].unique() + transformed_df = pd.DataFrame(primary_ag_product, columns=["name"]) + + logger.info(f"Successfully transformed data, found {len(transformed_df)} unique primary_ag_products.") + return transformed_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py deleted file mode 100644 index 237435cd..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional, Dict -import pandas as pd -from prefect import task, get_run_logger - -EXTRACT_SOURCES = ["basic_sample_info"] - -@task -def transform_products_primary_product(data_sources: Dict[str, pd.DataFrame]) -> Optional[pd.DataFrame]: - """ - Transforms the raw data to extract unique primary products. - - This function is purely for transformation (the 'T' in ETL). It takes the - raw DataFrame and performs the specific transformations needed for the - 'primary_product' table. - - Args: - data_sources: A dictionary of DataFrames from the extraction step. - - Returns: - A transformed DataFrame with a single 'Primary_crop' column containing - unique product names, or None if an error occurs. - """ - logger = get_run_logger() - logger.info("Transforming raw data for primary products...") - - raw_df = data_sources["basic_sample_info"] - - # Step 1: Check if the required column exists - if 'Primary_crop' not in raw_df.columns: - logger.error("'Primary_crop' column not found in the raw data.") - return None - - # Step 2: Get unique product names and create the final DataFrame - primary_product_names = raw_df['Primary_crop'].unique() - transformed_df = pd.DataFrame(primary_product_names, columns=["Primary_crop"]) - - logger.info(f"Successfully transformed data, found {len(transformed_df)} unique primary products.") - return transformed_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py new file mode 100644 index 00000000..7ae5cf9b --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py @@ -0,0 +1,26 @@ +from prefect import flow +from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info +from ca_biositing.pipeline.etl.transform.products.primary_ag_product import transform_products_primary_ag_product +from ca_biositing.pipeline.etl.load.products.primary_ag_product import load_products_primary_ag_product + +@flow(name="Primary Ag Product ETL", log_prints=True) +def primary_ag_product_flow(): + """ + ETL flow for processing primary agricultural products data. + + This flow extracts basic sample information, transforms it to identify + unique primary agricultural products, and loads them into the database. + """ + print("Running Primary Ag Product ETL flow...") + + # Extract + basic_sample_info_df = extract_basic_sample_info() + + # Transform + # The transform function expects a dictionary of data sources. + primary_ag_product_df = transform_products_primary_ag_product({"basic_sample_info": basic_sample_info_df}) + + # Load + load_products_primary_ag_product(primary_ag_product_df) + + print("Primary Ag Product ETL flow completed successfully.") diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py deleted file mode 100644 index 8e6a1524..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py +++ /dev/null @@ -1,26 +0,0 @@ -from prefect import flow -from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info -from ca_biositing.pipeline.etl.transform.products.primary_product import transform_products_primary_product -from ca_biositing.pipeline.etl.load.products.primary_product import load_products_primary_product - -@flow(name="Primary Product ETL", log_prints=True) -def primary_product_flow(): - """ - ETL flow for processing primary products. - - This flow extracts basic sample information, transforms it to identify - unique primary products, and loads them into the database. - """ - print("Running Primary Product ETL flow...") - - # Extract - basic_sample_info_df = extract_basic_sample_info() - - # Transform - # The transform function expects a dictionary of data sources. - primary_product_df = transform_products_primary_product({"basic_sample_info": basic_sample_info_df}) - - # Load - load_products_primary_product(primary_product_df) - - print("Primary Product ETL flow completed successfully.") From fd235be591c59c3b50d6a28c1d8af3566f896075 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 30 Dec 2025 08:55:42 -0700 Subject: [PATCH 3/7] holiday work. New notebook for importing gsheeets to pandas and playing around --- .../15467b7f2f3d_test_after_revert.py | 32 + .../linkml/modules/primary_ag_product.yaml | 23 + .../linkml/modules/resource_availability.yaml | 35 + .../linkml/modules/resource_class.yaml | 21 + .../modules/resource_counterfactual.yaml | 47 + .../linkml/modules/resource_morphology.yaml | 34 + .../linkml/modules/resource_subclass.yaml | 21 + .../schemas/generated/primary_ag_product.py | 75 ++ .../generated/resource_availability.py | 308 +++++ .../schemas/generated/resource_class.py | 74 ++ .../generated/resource_counterfactual.py | 1030 +++++++++++++++++ .../schemas/generated/resource_morphology.py | 308 +++++ .../schemas/generated/resource_subclass.py | 74 ++ .../datamodels/utils/database_joins.ipynb | 491 +------- .../pipeline/etl/extract/experiments.py | 2 +- .../pipeline/etl/extract/proximate.py | 59 + .../utils/gsheet_extraction_notebook.ipynb | 179 +++ 17 files changed, 2376 insertions(+), 437 deletions(-) create mode 100644 alembic/versions/15467b7f2f3d_test_after_revert.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb diff --git a/alembic/versions/15467b7f2f3d_test_after_revert.py b/alembic/versions/15467b7f2f3d_test_after_revert.py new file mode 100644 index 00000000..38dddb9d --- /dev/null +++ b/alembic/versions/15467b7f2f3d_test_after_revert.py @@ -0,0 +1,32 @@ +"""test after revert + +Revision ID: 15467b7f2f3d +Revises: e84690898528 +Create Date: 2025-12-25 05:02:50.212384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '15467b7f2f3d' +down_revision: Union[str, Sequence[str], None] = 'e84690898528' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml new file mode 100644 index 00000000..2c5c9b4c --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml @@ -0,0 +1,23 @@ +id: https://w3id.org/ca_biositing/primary_ag_product +name: primary_ag_product +title: Primary Agricultural Product +description: Core definitions for primary agricultural products. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + PrimaryAgProduct: + is_a: LookupBase + description: Primary agricultural product definition. + slots: + - note diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml new file mode 100644 index 00000000..9bf95246 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml @@ -0,0 +1,35 @@ +id: https://w3id.org/ca_biositing/resource_availability +name: resource_availability +title: Resource Availability +description: Availability of a resource in a location. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - places + - resource_information + +classes: + ResourceAvailability: + is_a: BaseEntity + description: Availability of a resource in a location. + slots: + - resource_id + - geoid + - from_month + - to_month + - year_round + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml new file mode 100644 index 00000000..cc39fe4e --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/resource_class +name: resource_class +title: Resource Class +description: Classification of resources. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + ResourceClass: + is_a: LookupBase + description: Classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml new file mode 100644 index 00000000..d346b197 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml @@ -0,0 +1,47 @@ +id: https://w3id.org/ca_biositing/resource_counterfactual +name: resource_counterfactual +title: Resource Counterfactual +description: Counterfactual uses of a resource. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - places + - resource_information + - data_sources_metadata + +classes: + ResourceCounterfactual: + is_a: BaseEntity + description: Counterfactual uses of a resource. + slots: + - geoid + - resource_id + - counterfactual_description + - animal_bedding_percent + - animal_bedding_source_id + - animal_feed_percent + - animal_feed_source_id + - bioelectricty_percent + - bioelectricty_source_id + - burn_percent + - burn_source_id + - compost_percent + - compost_source_id + - landfill_percent + - landfill_source_id + - counterfactual_date + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml new file mode 100644 index 00000000..f9d93c67 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml @@ -0,0 +1,34 @@ +id: https://w3id.org/ca_biositing/resource_morphology +name: resource_morphology +title: Resource Morphology +description: Morphology of a resource. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - resource_information + +classes: + ResourceMorphology: + description: Morphology of a resource. + slots: + - id + - resource_id + - morphology_uri + slot_usage: + id: + identifier: true + range: integer + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml new file mode 100644 index 00000000..0236bca8 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/resource_subclass +name: resource_subclass +title: Resource Subclass +description: Sub-classification of resources. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + ResourceSubclass: + is_a: LookupBase + description: Sub-classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py new file mode 100644 index 00000000..dfa0f14f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py @@ -0,0 +1,75 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py new file mode 100644 index 00000000..c02c738f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py @@ -0,0 +1,308 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py new file mode 100644 index 00000000..b47201f8 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py @@ -0,0 +1,74 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py new file mode 100644 index 00000000..4568cca1 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py @@ -0,0 +1,1030 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class ParameterCategoryParameter(Base): + """ + Link between Parameter and ParameterCategory. + """ + __tablename__ = 'parameter_category_parameter' + + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer()) + parameter_category_id = Column(Integer()) + + + def __repr__(self): + return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" + + + + + + +class ParameterUnit(Base): + """ + Link between Parameter and Unit (alternate units). + """ + __tablename__ = 'parameter_unit' + + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer()) + alternate_unit_id = Column(Integer()) + + + def __repr__(self): + return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" + + + + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Contact(BaseEntity): + """ + Contact information for a person. + """ + __tablename__ = 'contact' + + first_name = Column(Text()) + last_name = Column(Text()) + email = Column(Text()) + affiliation = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Provider(BaseEntity): + """ + Provider information. + """ + __tablename__ = 'provider' + + codename = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class DataSource(BaseEntity): + """ + Source of data. + """ + __tablename__ = 'data_source' + + name = Column(Text()) + description = Column(Text()) + data_source_type_id = Column(Integer()) + full_title = Column(Text()) + creator = Column(Text()) + subject = Column(Text()) + publisher = Column(Text()) + contributor = Column(Text()) + date = Column(DateTime()) + type = Column(Text()) + biocirv = Column(Boolean()) + format = Column(Text()) + language = Column(Text()) + relation = Column(Text()) + temporal_coverage = Column(Text()) + location_coverage_id = Column(Integer()) + rights = Column(Text()) + license = Column(Text()) + uri = Column(Text()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FileObjectMetadata(BaseEntity): + """ + Metadata for a file object. + """ + __tablename__ = 'file_object_metadata' + + data_source_id = Column(Integer()) + bucket_path = Column(Text()) + file_format = Column(Text()) + file_size = Column(Integer()) + checksum_md5 = Column(Text()) + checksum_sha256 = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class DataSourceType(BaseEntity): + """ + Type of data source (e.g. database, literature). + """ + __tablename__ = 'data_source_type' + + source_type_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationResolution(LookupBase): + """ + Resolution of the location (e.g. nation, state, county). + """ + __tablename__ = 'location_resolution' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class SourceType(LookupBase): + """ + Type of source (e.g. database, literature). + """ + __tablename__ = 'source_type' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Unit(LookupBase): + """ + Unit of measurement. + """ + __tablename__ = 'unit' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Method(BaseEntity): + """ + Analytical method. + """ + __tablename__ = 'method' + + name = Column(Text()) + method_abbrev_id = Column(Integer()) + method_category_id = Column(Integer()) + method_standard_id = Column(Integer()) + description = Column(Text()) + detection_limits = Column(Text()) + source_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodAbbrev(LookupBase): + """ + Abbreviation for method. + """ + __tablename__ = 'method_abbrev' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodCategory(LookupBase): + """ + Category of method. + """ + __tablename__ = 'method_category' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodStandard(LookupBase): + """ + Standard associated with the method. + """ + __tablename__ = 'method_standard' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Parameter(BaseEntity): + """ + Parameter being measured. + """ + __tablename__ = 'parameter' + + name = Column(Text()) + standard_unit_id = Column(Integer()) + calculated = Column(Boolean()) + description = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ParameterCategory(LookupBase): + """ + Category of parameter. + """ + __tablename__ = 'parameter_category' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldSample(BaseEntity): + """ + Sample collected from the field. + """ + __tablename__ = 'field_sample' + + name = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + provider_id = Column(Integer()) + collector_id = Column(Integer()) + sample_collection_source = Column(Text()) + amount_collected = Column(Numeric()) + amount_collected_unit_id = Column(Integer()) + sampling_location_id = Column(Integer()) + field_storage_method_id = Column(Integer()) + field_storage_duration_value = Column(Numeric()) + field_storage_duration_unit_id = Column(Integer()) + field_storage_location_id = Column(Integer()) + collection_timestamp = Column(DateTime()) + collection_method_id = Column(Integer()) + harvest_method_id = Column(Integer()) + harvest_date = Column(Date()) + field_sample_storage_location_id = Column(Integer()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PhysicalCharacteristic(BaseEntity): + """ + Physical characteristics of a sample. + """ + __tablename__ = 'physical_characteristic' + + field_sample_id = Column(Integer()) + particle_length = Column(Numeric()) + particle_width = Column(Numeric()) + particle_height = Column(Numeric()) + particle_unit_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldSampleCondition(BaseEntity): + """ + Condition of the field sample. + """ + __tablename__ = 'field_sample_condition' + + field_sample_id = Column(Integer()) + ag_treatment_id = Column(Integer()) + last_application_date = Column(Date()) + treatment_amount_per_acre = Column(Float()) + processing_method_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldStorageMethod(LookupBase): + """ + Method of field storage. + """ + __tablename__ = 'field_storage_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class CollectionMethod(LookupBase): + """ + Method of collection. + """ + __tablename__ = 'collection_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class HarvestMethod(LookupBase): + """ + Method of harvest. + """ + __tablename__ = 'harvest_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ProcessingMethod(LookupBase): + """ + Method of processing. + """ + __tablename__ = 'processing_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class SoilType(LookupBase): + """ + Type of soil. + """ + __tablename__ = 'soil_type' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class AgTreatment(LookupBase): + """ + Agricultural treatment. + """ + __tablename__ = 'ag_treatment' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationSoilType(BaseEntity): + """ + Soil type at a location. + """ + __tablename__ = 'location_soil_type' + + location_id = Column(Integer()) + soil_type_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py new file mode 100644 index 00000000..0b9bcf94 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py @@ -0,0 +1,308 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py new file mode 100644 index 00000000..a3919852 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py @@ -0,0 +1,74 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/utils/database_joins.ipynb b/src/ca_biositing/datamodels/utils/database_joins.ipynb index dbb836f2..e4a9b9a5 100644 --- a/src/ca_biositing/datamodels/utils/database_joins.ipynb +++ b/src/ca_biositing/datamodels/utils/database_joins.ipynb @@ -15,21 +15,13 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Python Executable: /Users/pjsmitty301/ca-biositing/.pixi/envs/default/bin/python\n", - "✅ You are running in a Pixi environment.\n" - ] - } - ], + "outputs": [], "source": [ "# Verify Kernel\n", "import sys\n", + "\n", "print(f\"Python Executable: {sys.executable}\")\n", "\n", "# Check if we are in the pixi environment (path should contain .pixi)\n", @@ -41,17 +33,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Libraries and models imported successfully.\n" - ] - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "from sqlalchemy import create_engine, select\n", @@ -65,17 +49,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected to database.\n" - ] - } - ], + "outputs": [], "source": [ "# Database Connection\n", "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", @@ -96,22 +72,23 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": " is not a generic class", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m stmt2 = select(LandiqRecord, PrimaryCrop).join(\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mPrimaryCrop\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m, \n\u001b[32m 3\u001b[39m LandiqRecord.main_crop == PrimaryCrop.id).limit(\u001b[32m10\u001b[39m)\n\u001b[32m 5\u001b[39m \u001b[38;5;28mprint\u001b[39m(stmt2)\n\u001b[32m 7\u001b[39m df2 = pd.read_sql(stmt2, engine)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/typing.py:398\u001b[39m, in \u001b[36m_tp_cache..decorator..inner\u001b[39m\u001b[34m(*args, **kwds)\u001b[39m\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[32m 397\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m \u001b[38;5;66;03m# All real errors (not unhashable args) are raised below.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m398\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/typing.py:1110\u001b[39m, in \u001b[36m_generic_class_getitem\u001b[39m\u001b[34m(cls, params)\u001b[39m\n\u001b[32m 1108\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m prepare \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1109\u001b[39m params = prepare(\u001b[38;5;28mcls\u001b[39m, params)\n\u001b[32m-> \u001b[39m\u001b[32m1110\u001b[39m \u001b[43m_check_generic\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m__parameters__\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1112\u001b[39m new_args = []\n\u001b[32m 1113\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m param, new_arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;28mcls\u001b[39m.__parameters__, params):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/site-packages/typing_extensions.py:3111\u001b[39m, in \u001b[36m_check_generic\u001b[39m\u001b[34m(cls, parameters, elen)\u001b[39m\n\u001b[32m 3106\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"Check correct count for parameters of a generic cls (internal helper).\u001b[39;00m\n\u001b[32m 3107\u001b[39m \n\u001b[32m 3108\u001b[39m \u001b[33;03mThis gives a nice error message in case of count mismatch.\u001b[39;00m\n\u001b[32m 3109\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 3110\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m elen:\n\u001b[32m-> \u001b[39m\u001b[32m3111\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is not a generic class\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 3112\u001b[39m alen = \u001b[38;5;28mlen\u001b[39m(parameters)\n\u001b[32m 3113\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m alen != elen:\n", - "\u001b[31mTypeError\u001b[39m: is not a generic class" - ] - } - ], + "outputs": [], + "source": [ + "# Create the join query\n", + "stmt = select(LandiqRecord, Observation).join(\n", + " Observation,\n", + " Observation.record_id == LandiqRecord.id\n", + ")\n", + "\n", + "print(\"SQL Query:\")\n", + "print(stmt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "stmt2 = select(LandiqRecord, PrimaryCrop).join(\n", " PrimaryCrop, \n", @@ -124,281 +101,11 @@ "df2.columns\n" ] }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SQL Query:\n", - "SELECT landiq_record.dataset_id, landiq_record.polygon_id, landiq_record.main_crop, landiq_record.secondary_crop, landiq_record.tertiary_crop, landiq_record.quaternary_crop, landiq_record.confidence, landiq_record.irrigated, landiq_record.acres, landiq_record.version, landiq_record.note, landiq_record.test, landiq_record.id, landiq_record.created_at, landiq_record.updated_at, landiq_record.etl_run_id, landiq_record.lineage_group_id, observation.dataset_id AS dataset_id_1, observation.record_type, observation.record_id, observation.parameter_id, observation.value, observation.unit_id, observation.dimension_type_id, observation.dimension_value, observation.dimension_unit_id, observation.note AS note_1, observation.id AS id_1, observation.created_at AS created_at_1, observation.updated_at AS updated_at_1, observation.etl_run_id AS etl_run_id_1, observation.lineage_group_id AS lineage_group_id_1 \n", - "FROM landiq_record JOIN observation ON observation.record_id = landiq_record.id\n" - ] - } - ], - "source": [ - "# Create the join query\n", - "stmt = select(LandiqRecord, Observation).join(\n", - " Observation,\n", - " Observation.record_id == LandiqRecord.id\n", - ")\n", - "\n", - "print(\"SQL Query:\")\n", - "print(stmt)" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded 0 rows.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "dataset_id", - "rawType": "object", - "type": "string" - }, - { - "name": "polygon_id", - "rawType": "object", - "type": "string" - }, - { - "name": "main_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "secondary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "tertiary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "quaternary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "confidence", - "rawType": "object", - "type": "string" - }, - { - "name": "irrigated", - "rawType": "object", - "type": "string" - }, - { - "name": "acres", - "rawType": "object", - "type": "string" - }, - { - "name": "version", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "string" - }, - { - "name": "test", - "rawType": "object", - "type": "string" - }, - { - "name": "id", - "rawType": "object", - "type": "string" - }, - { - "name": "created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "etl_run_id", - "rawType": "object", - "type": "string" - }, - { - "name": "lineage_group_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dataset_id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "record_type", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "object", - "type": "string" - }, - { - "name": "unit_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_type_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_value", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_unit_id", - "rawType": "object", - "type": "string" - }, - { - "name": "note_1", - "rawType": "object", - "type": "string" - }, - { - "name": "id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "created_at_1", - "rawType": "object", - "type": "string" - }, - { - "name": "updated_at_1", - "rawType": "object", - "type": "string" - }, - { - "name": "etl_run_id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "lineage_group_id_1", - "rawType": "object", - "type": "string" - } - ], - "ref": "edc059e1-7169-4eb9-ad76-c55b46d93053", - "rows": [], - "shape": { - "columns": 32, - "rows": 0 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dataset_idpolygon_idmain_cropsecondary_croptertiary_cropquaternary_cropconfidenceirrigatedacresversion...unit_iddimension_type_iddimension_valuedimension_unit_idnote_1id_1created_at_1updated_at_1etl_run_id_1lineage_group_id_1
\n", - "

0 rows × 32 columns

\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [dataset_id, polygon_id, main_crop, secondary_crop, tertiary_crop, quaternary_crop, confidence, irrigated, acres, version, note, test, id, created_at, updated_at, etl_run_id, lineage_group_id, dataset_id_1, record_type, record_id, parameter_id, value, unit_id, dimension_type_id, dimension_value, dimension_unit_id, note_1, id_1, created_at_1, updated_at_1, etl_run_id_1, lineage_group_id_1]\n", - "Index: []\n", - "\n", - "[0 rows x 32 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Execute and load into DataFrame\n", "try:\n", @@ -415,135 +122,47 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['dataset_id', 'polygon_id', 'main_crop', 'secondary_crop',\n", - " 'tertiary_crop', 'quaternary_crop', 'confidence', 'irrigated', 'acres',\n", - " 'version', 'note', 'test', 'id', 'created_at', 'updated_at',\n", - " 'etl_run_id', 'lineage_group_id', 'dataset_id_1', 'record_type',\n", - " 'record_id', 'parameter_id', 'value', 'unit_id', 'dimension_type_id',\n", - " 'dimension_value', 'dimension_unit_id', 'note_1', 'id_1',\n", - " 'created_at_1', 'updated_at_1', 'etl_run_id_1', 'lineage_group_id_1'],\n", - " dtype='object')" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.columns" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "id", - "rawType": "object", - "type": "string" - }, - { - "name": "dataset_id", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "main_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "object", - "type": "string" - }, - { - "name": "unit_id", - "rawType": "object", - "type": "string" - } - ], - "ref": "5c95c323-56ec-4546-ad20-84e7b50c7748", - "rows": [], - "shape": { - "columns": 7, - "rows": 0 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
iddataset_idrecord_idmain_cropparameter_idvalueunit_id
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [id, dataset_id, record_id, main_crop, parameter_id, value, unit_id]\n", - "Index: []" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[[\"id\", \"dataset_id\", \"record_id\", \"main_crop\", \"parameter_id\", \"value\", \"unit_id\"]]" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = select(Observation, LandiqRecord, PrimaryCrop).join(\n", + " Observation,\n", + " Observation.record_id == LandiqRecord.id).join(\n", + " PrimaryCrop,\n", + " LandiqRecord.main_crop == PrimaryCrop.id).limit(10)\n", + "\n", + "df3 = pd.read_sql(query, engine)\n", + "\n", + "df3.columns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.read_sql(select(LandiqRecord), engine)" + ] } ], "metadata": { diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py index 25a6d194..9a74e350 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py @@ -1,7 +1,7 @@ from typing import Optional import pandas as pd from prefect import task, get_run_logger -from ca_biositing.pipeline.utils.gsheet_to_pandas import gsheet_to_df +from ...utils.gsheet_to_pandas import gsheet_to_df @task def extract_experiments() -> Optional[pd.DataFrame]: diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py new file mode 100644 index 00000000..9bbaf910 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.1-Proximate" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb new file mode 100644 index 00000000..b098f40d --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import janitor as jn\n", + "from IPython.display import display\n", + "\n", + "# --- Robustly find the project root ---\n", + "# The project root is the directory containing the 'pixi.toml' file.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path): # Stop at the filesystem root\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# --- Add project root to sys.path ---\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " print(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " print(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import the module ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate\n", + " print(\"Successfully imported 'proximate' module.\")\n", + "except ImportError as e:\n", + " print(f\"Failed to import 'proximate' module: {e}\")\n", + " print(f\"\\nFull sys.path: {sys.path}\")\n", + "\n", + "# --- Run the extraction ---\n", + "if 'proximate' in locals():\n", + " try:\n", + " # Pass the project_root to the extract function\n", + " df = proximate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted data.\")\n", + " display(df.head())\n", + " else:\n", + " print(\"\\nExtraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during extraction: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.clean_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df[['Record_ID', 'Source_codename', 'Prepared_sample']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Convert 'Value' column to numeric\n", + "pv_df = df[['Parameter', 'Value']]\n", + "\n", + "pv_df['Value'] = pd.to_numeric(pv_df['Value'], errors='coerce')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pv_df\n", + "\n", + "if pv_df.groupby('Parameter').mean() is not None:\n", + " print(\"Mean values by Parameter:\")\n", + " display(pv_df.groupby('Parameter').mean())\n", + "else:\n", + " print(\"No numeric values available to compute means.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = df[['Resource', 'Parameter', 'Value', 'Unit']]\n", + "\n", + "df2['Value'] = pd.to_numeric(df2['Value'], errors='coerce')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = df2.dropna(subset=['Value'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats = df2.groupby(['Resource', 'Parameter'])['Value'].agg(['mean', 'median', 'min', 'max', 'std', 'count'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 907686ba1ca492efd578895d1eb825ea9e7ea961 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Fri, 2 Jan 2026 21:41:29 -0700 Subject: [PATCH 4/7] feat: notebook for gsheets extraction, data playground, and pk id lookup insertion. --- .../pipeline/etl/extract/cmpana.py | 59 + .../pipeline/etl/extract/ultimate.py | 59 + .../ca_biositing/pipeline/utils/engine.py | 6 + .../utils/gsheet_extraction_notebook.ipynb | 4100 ++++++++++++++++- .../pipeline/utils/name_id_swap.py | 81 + .../pipeline/utils/seed_biomass.sql | 15 - 6 files changed, 4272 insertions(+), 48 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py new file mode 100644 index 00000000..b34a39ca --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.3-CmpAna" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py new file mode 100644 index 00000000..c7495876 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.7-Ultimate" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py new file mode 100644 index 00000000..a24e92a0 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py @@ -0,0 +1,6 @@ +from sqlmodel import create_engine, Session + +DATABASE_URL = "postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db" +engine = create_engine(DATABASE_URL) + +db_session = Session(engine) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb index b098f40d..9d27799e 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -2,13 +2,1522 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", + "Successfully imported all module.\n" + ] + }, + { + "data": { + "text/html": [ + "
20:54:42.753 | INFO    | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:42.753 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.230 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.230 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.234 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.234 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted proximate data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Prox_UUID_031", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Source_codename", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + } + ], + "ref": "b6c1f313-06e4-4dba-8a9e-6a1dc878820f", + "rows": [ + [ + "0", + "D7965110-407F-E356-D41D-B3B9A2B7B7", + "(73)B7B7", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "1", + "Prox01xk(73)1", + "Moisture", + "61.85", + "% total weight", + "2024-10-02 10:31:01", + "", + "Pass", + "not ready", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "1", + "C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8", + "(73)9BA8", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "2", + "Prox01xk(73)2", + "Moisture", + "63.21", + "% total weight", + "2024-10-02 10:31:31", + "", + "Pass", + "ready", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "2", + "DF304D5D-3A85-4881-7142-6D4E5F957D", + "(73)957D", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "3", + "Prox01xk(73)3", + "Moisture", + "63.27", + "% total weight", + "2024-10-02 10:32:01", + "", + "Pass", + "imported", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "3", + "01C6C5BE-CEA6-54AF-3924-B0BAD69335", + "(73)9335", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "1", + "Prox01xk(73)1", + "Ash", + "0.69", + "% total weight", + "2024-10-03 10:31:01", + "", + "Pass", + "import failed", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "4", + "126745C7-DD41-2F6D-0DC5-28DBCA415F", + "(73)415F", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "2", + "Prox01xk(73)2", + "Ash", + "0.89", + "% total weight", + "2024-10-03 10:31:31", + "", + "Pass", + "", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ] + ], + "shape": { + "columns": 20, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Prox_UUID_031Record_IDSource_codenamePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameterValueUnitCreated_atUpdated_atQC_resultUpload_statusNoteAnalysis_typeAnalyst_email
0D7965110-407F-E356-D41D-B3B9A2B7B7(73)B7B7OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk1Prox01xk(73)1Moisture61.85% total weight2024-10-02 10:31:01Passnot readyProximate analysisxkang2@lbl.gov
1C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8(73)9BA8OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk2Prox01xk(73)2Moisture63.21% total weight2024-10-02 10:31:31PassreadyProximate analysisxkang2@lbl.gov
2DF304D5D-3A85-4881-7142-6D4E5F957D(73)957DOakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk3Prox01xk(73)3Moisture63.27% total weight2024-10-02 10:32:01PassimportedProximate analysisxkang2@lbl.gov
301C6C5BE-CEA6-54AF-3924-B0BAD69335(73)9335OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk1Prox01xk(73)1Ash0.69% total weight2024-10-03 10:31:01Passimport failedProximate analysisxkang2@lbl.gov
4126745C7-DD41-2F6D-0DC5-28DBCA415F(73)415FOakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk2Prox01xk(73)2Ash0.89% total weight2024-10-03 10:31:31PassProximate analysisxkang2@lbl.gov
\n", + "
" + ], + "text/plain": [ + " Prox_UUID_031 Record_ID Source_codename \\\n", + "0 D7965110-407F-E356-D41D-B3B9A2B7B7 (73)B7B7 Oakleaf \n", + "1 C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8 (73)9BA8 Oakleaf \n", + "2 DF304D5D-3A85-4881-7142-6D4E5F957D (73)957D Oakleaf \n", + "3 01C6C5BE-CEA6-54AF-3924-B0BAD69335 (73)9335 Oakleaf \n", + "4 126745C7-DD41-2F6D-0DC5-28DBCA415F (73)415F Oakleaf \n", + "\n", + " Prepared_sample Resource Preparation_method Storage_cond \\\n", + "0 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "1 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "2 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "3 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "4 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "\n", + " Exper_abbrev Repl_no Repl_ID Parameter Value Unit \\\n", + "0 Prox01xk 1 Prox01xk(73)1 Moisture 61.85 % total weight \n", + "1 Prox01xk 2 Prox01xk(73)2 Moisture 63.21 % total weight \n", + "2 Prox01xk 3 Prox01xk(73)3 Moisture 63.27 % total weight \n", + "3 Prox01xk 1 Prox01xk(73)1 Ash 0.69 % total weight \n", + "4 Prox01xk 2 Prox01xk(73)2 Ash 0.89 % total weight \n", + "\n", + " Created_at Updated_at QC_result Upload_status Note \\\n", + "0 2024-10-02 10:31:01 Pass not ready \n", + "1 2024-10-02 10:31:31 Pass ready \n", + "2 2024-10-02 10:32:01 Pass imported \n", + "3 2024-10-03 10:31:01 Pass import failed \n", + "4 2024-10-03 10:31:31 Pass \n", + "\n", + " Analysis_type Analyst_email \n", + "0 Proximate analysis xkang2@lbl.gov \n", + "1 Proximate analysis xkang2@lbl.gov \n", + "2 Proximate analysis xkang2@lbl.gov \n", + "3 Proximate analysis xkang2@lbl.gov \n", + "4 Proximate analysis xkang2@lbl.gov " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.272 | INFO    | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.272 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.593 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.593 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.595 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.595 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted Ultimate data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Ult_UUID_037", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Ult_sample_name", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "Raw_data_URL", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + } + ], + "ref": "61714e59-0541-4b74-9d38-a37794cbd34d", + "rows": [ + [ + "0", + "421617A5-E50E-9642-2974-B3275FE822", + ")U22E822", + "Hum-AlmHu023KM2(15)U22", + "Hum-AlmHu023KM2(15)", + "Almond Hulls", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(15)1", + "DM", + "8.86E+01", + "pc", + "", + "", + "Pass", + "1", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "1", + "7E7919C2-5DB4-6BEF-75E2-7E51321200", + ")U001200", + "Hum-AlmHu023KM2(15)U00", + "Hum-AlmHu023KM2(15)", + "Almond Hulls", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(15)1", + "DM", + "", + "pc", + "", + "", + "Fail", + " 1 dup", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "2", + "3AA85881-1185-642F-C44B-41AD2275D2", + ")UD275D2", + "Hum-AlmSh022KM2(13)UD2", + "Hum-AlmSh022KM2(13)", + "Almond Shells", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(13)1", + "DM", + "8.13E+01", + "pc", + "", + "", + "Pass", + "2", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "3", + "FA418804-6C4F-4C90-D78F-84D7DF54D3", + ")UD354D3", + "Ene-WaSh017OKM2(82)UD3", + "Ene-WaSh017OKM2(82)", + "Walnut Shells", + "Oven Dry + Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(82)1", + "DM", + "9.22E+01", + "pc", + "", + "", + "Pass", + "3", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "4", + "6FDACBFC-7E0B-473B-444F-85B7650267", + ")U670267", + "Ebo-GpPm010OKM2(1B)U67", + "Ebo-GpPm010OKM2(1B)", + "Grape pomace", + "Oven Dry + Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(1B)1", + "DM", + "9.38E+01", + "pc", + "", + "", + "Pass", + "4", + "Ultimate analysis", + "", + "", + "", + "" + ] + ], + "shape": { + "columns": 22, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Ult_UUID_037Record_IDUlt_sample_namePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_ID...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
0421617A5-E50E-9642-2974-B3275FE822)U22E822Hum-AlmHu023KM2(15)U22Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcPass1Ultimate analysis
17E7919C2-5DB4-6BEF-75E2-7E51321200)U001200Hum-AlmHu023KM2(15)U00Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcFail1 dupUltimate analysis
23AA85881-1185-642F-C44B-41AD2275D2)UD275D2Hum-AlmSh022KM2(13)UD2Hum-AlmSh022KM2(13)Almond ShellsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(13)1...pcPass2Ultimate analysis
3FA418804-6C4F-4C90-D78F-84D7DF54D3)UD354D3Ene-WaSh017OKM2(82)UD3Ene-WaSh017OKM2(82)Walnut ShellsOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(82)1...pcPass3Ultimate analysis
46FDACBFC-7E0B-473B-444F-85B7650267)U670267Ebo-GpPm010OKM2(1B)U67Ebo-GpPm010OKM2(1B)Grape pomaceOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(1B)1...pcPass4Ultimate analysis
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " Ult_UUID_037 Record_ID Ult_sample_name \\\n", + "0 421617A5-E50E-9642-2974-B3275FE822 )U22E822 Hum-AlmHu023KM2(15)U22 \n", + "1 7E7919C2-5DB4-6BEF-75E2-7E51321200 )U001200 Hum-AlmHu023KM2(15)U00 \n", + "2 3AA85881-1185-642F-C44B-41AD2275D2 )UD275D2 Hum-AlmSh022KM2(13)UD2 \n", + "3 FA418804-6C4F-4C90-D78F-84D7DF54D3 )UD354D3 Ene-WaSh017OKM2(82)UD3 \n", + "4 6FDACBFC-7E0B-473B-444F-85B7650267 )U670267 Ebo-GpPm010OKM2(1B)U67 \n", + "\n", + " Prepared_sample Resource Preparation_method \\\n", + "0 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", + "1 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", + "2 Hum-AlmSh022KM2(13) Almond Shells Knife Mill (2mm) \n", + "3 Ene-WaSh017OKM2(82) Walnut Shells Oven Dry + Knife Mill (2mm) \n", + "4 Ebo-GpPm010OKM2(1B) Grape pomace Oven Dry + Knife Mill (2mm) \n", + "\n", + " Storage_cond Exper_abbrev Repl_no Repl_ID ... Unit Created_at \\\n", + "0 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", + "1 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", + "2 RT vacuum sealed Ult26kh 1 Ult26kh(13)1 ... pc \n", + "3 RT vacuum sealed Ult26kh 1 Ult26kh(82)1 ... pc \n", + "4 RT vacuum sealed Ult26kh 1 Ult26kh(1B)1 ... pc \n", + "\n", + " Updated_at QC_result Note Analysis_type Equipment Raw_data_URL \\\n", + "0 Pass 1 Ultimate analysis \n", + "1 Fail 1 dup Ultimate analysis \n", + "2 Pass 2 Ultimate analysis \n", + "3 Pass 3 Ultimate analysis \n", + "4 Pass 4 Ultimate analysis \n", + "\n", + " Analyst_email Upload_status \n", + "0 \n", + "1 \n", + "2 \n", + "3 \n", + "4 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.631 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.631 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:47.211 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:47.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:47.214 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:47.214 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted CmpAna data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Cmp_UUID_033", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "Raw_data_URL", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + } + ], + "ref": "2dd03f9d-30e4-4e23-af07-409bfdca5047", + "rows": [ + [ + "0", + "3EE2993D-86E3-1F16-C7EA-F8D555E114", + "(85)E114", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "Glucan", + "14.16", + "% dry weight", + "1/23/2025 9:00:01", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "1", + "46878EF9-1226-22A0-D5D8-CF65E241CB", + "(85)41CB", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "Glucan", + "14.18", + "% dry weight", + "1/23/2025 9:00:16", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "2", + "76A7A2F4-C4E4-E60F-1187-DEC6E02246", + "(85)2246", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "Glucan", + "14.12", + "% dry weight", + "1/23/2025 9:00:31", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "3", + "7A136832-286B-07CB-62DE-ACF52F9311", + "(85)9311", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "Glucose", + "15.74", + "% dry weight", + "1/23/2025 9:00:46", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "4", + "B709ECEE-F9A6-A55D-A59E-93B7B863D7", + "(85)63D7", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "Glucose", + "15.75", + "% dry weight", + "1/23/2025 9:01:01", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ] + ], + "shape": { + "columns": 21, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Cmp_UUID_033Record_IDPrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameter...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucan...% dry weight1/23/2025 9:00:01passChemical compositionxkang2@lbl.govready
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucan...% dry weight1/23/2025 9:00:16passChemical compositionxkang2@lbl.govready
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)3Glucan...% dry weight1/23/2025 9:00:31passChemical compositionxkang2@lbl.govready
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucose...% dry weight1/23/2025 9:00:46passChemical compositionxkang2@lbl.govready
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucose...% dry weight1/23/2025 9:01:01passChemical compositionxkang2@lbl.govready
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " Cmp_UUID_033 Record_ID Prepared_sample \\\n", + "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", + "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", + "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", + "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", + "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", + "\n", + " Resource Preparation_method Storage_cond Exper_abbrev Repl_no \\\n", + "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", + "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", + "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 3 \n", + "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", + "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", + "\n", + " Repl_ID Parameter ... Unit Created_at Updated_at \\\n", + "0 Cmp04xk(85)1 Glucan ... % dry weight 1/23/2025 9:00:01 \n", + "1 Cmp04xk(85)2 Glucan ... % dry weight 1/23/2025 9:00:16 \n", + "2 Cmp04xk(85)3 Glucan ... % dry weight 1/23/2025 9:00:31 \n", + "3 Cmp04xk(85)1 Glucose ... % dry weight 1/23/2025 9:00:46 \n", + "4 Cmp04xk(85)2 Glucose ... % dry weight 1/23/2025 9:01:01 \n", + "\n", + " QC_result Note Analysis_type Equipment Raw_data_URL Analyst_email \\\n", + "0 pass Chemical composition xkang2@lbl.gov \n", + "1 pass Chemical composition xkang2@lbl.gov \n", + "2 pass Chemical composition xkang2@lbl.gov \n", + "3 pass Chemical composition xkang2@lbl.gov \n", + "4 pass Chemical composition xkang2@lbl.gov \n", + "\n", + " Upload_status \n", + "0 ready \n", + "1 ready \n", + "2 ready \n", + "3 ready \n", + "4 ready \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Cmp Ana extraction process completed.\n" + ] + } + ], "source": [ "import os\n", "import sys\n", "import pandas as pd\n", + "import numpy as np\n", "import janitor as jn\n", "from IPython.display import display\n", "\n", @@ -34,10 +1543,10 @@ "\n", "# --- Import the module ---\n", "try:\n", - " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate\n", - " print(\"Successfully imported 'proximate' module.\")\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " print(\"Successfully imported all module.\")\n", "except ImportError as e:\n", - " print(f\"Failed to import 'proximate' module: {e}\")\n", + " print(f\"Failed to import modules: {e}\")\n", " print(f\"\\nFull sys.path: {sys.path}\")\n", "\n", "# --- Run the extraction ---\n", @@ -46,21 +1555,67 @@ " # Pass the project_root to the extract function\n", " df = proximate.extract(project_root=project_root)\n", " if df is not None:\n", - " print(\"\\nSuccessfully extracted data.\")\n", + " print(\"\\nSuccessfully extracted proximate data.\")\n", " display(df.head())\n", " else:\n", - " print(\"\\nExtraction returned no data. Check the logs above for errors.\")\n", + " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", + "\n", + "if 'ultimate' in locals():\n", + " try:\n", + " df2 = ultimate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted Ultimate data.\")\n", + " display(df2.head())\n", + " else:\n", + " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", " except Exception as e:\n", - " print(f\"\\nAn error occurred during extraction: {e}\")" + " print(f\"\\nAn error occurred during extraction: {e}\")\n", + "\n", + "if 'cmpana' in locals():\n", + " try:\n", + " df3 = cmpana.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted CmpAna data.\")\n", + " display(df3.head())\n", + " else:\n", + " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", + " finally:\n", + " print(\"\\nCmp Ana extraction process completed.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "df.clean_names()" + "def clean_the_gsheets(df):\n", + " # 1. Clean names and drop rows\n", + " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + "\n", + " # 2. Coerce types (using errors='coerce' handles messy string data)\n", + " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", + " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Dates\n", + " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", + " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert remaining objects to best possible types (like strings)\n", + " df = df.convert_dtypes()\n", + " \n", + " return df # Return the FULL dataframe, not just .head()" ] }, { @@ -69,7 +1624,11 @@ "metadata": {}, "outputs": [], "source": [ - "df[['Record_ID', 'Source_codename', 'Prepared_sample']]" + "dataframes = [df, df2, df3]\n", + "\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", + "\n", + "clean_dataframes[2].head()\n" ] }, { @@ -78,10 +1637,11 @@ "metadata": {}, "outputs": [], "source": [ - "## Convert 'Value' column to numeric\n", - "pv_df = df[['Parameter', 'Value']]\n", + "summary_stats = clean_dataframes[0].\\\n", + " groupby(['resource', 'parameter'])['value'].\\\n", + " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", "\n", - "pv_df['Value'] = pd.to_numeric(pv_df['Value'], errors='coerce')" + "summary_stats" ] }, { @@ -90,13 +1650,12 @@ "metadata": {}, "outputs": [], "source": [ - "pv_df\n", - "\n", - "if pv_df.groupby('Parameter').mean() is not None:\n", - " print(\"Mean values by Parameter:\")\n", - " display(pv_df.groupby('Parameter').mean())\n", - "else:\n", - " print(\"No numeric values available to compute means.\")" + "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", + " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", + " agg({'value': 'mean'}).\\\n", + " query('value > 30').\\\n", + " sort_values(by='value', ascending=False).\\\n", + " round({'value': 1})\n" ] }, { @@ -105,9 +1664,18 @@ "metadata": {}, "outputs": [], "source": [ - "df2 = df[['Resource', 'Parameter', 'Value', 'Unit']]\n", + "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", + "\n", + "def is_it_volatile_solids(df):\n", + " df['check'] = \"VS\"\n", + "\n", + " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", + " return df\n", "\n", - "df2['Value'] = pd.to_numeric(df2['Value'], errors='coerce')" + "is_it_volatile_solids(df)\n", + "\n", + "df[['check', 'parameter']]\n", + "\n" ] }, { @@ -116,7 +1684,17 @@ "metadata": {}, "outputs": [], "source": [ - "df2" + "\n", + "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", + "def sqrtvalue(df):\n", + " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", + " return df\n", + "\n", + "#List comprehension to apply sqrtvalue to each DataFrame\n", + "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", + "\n", + "# Display the head of the third DataFrame\n", + "clean_rooted_df[2].head()" ] }, { @@ -125,25 +1703,2462 @@ "metadata": {}, "outputs": [], "source": [ - "df2 = df2.dropna(subset=['Value'])" + "cmpana_raw = cmpana.extract(project_root=project_root)\n", + "\n", + "cmpana_raw.head()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connected to database.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "0dabe9e2-728a-486d-9dd1-c1edd094e9c2", + "rows": [ + [ + "0", + "1", + "Tomatoes for processing", + null, + null, + null + ], + [ + "1", + "2", + "Grapes", + null, + null, + null + ], + [ + "2", + "3", + "Almonds", + null, + null, + null + ], + [ + "3", + "4", + "Walnuts", + null, + null, + null + ], + [ + "4", + "5", + "Sweet potatoes", + null, + null, + null + ], + [ + "5", + "6", + "Algae", + null, + null, + null + ], + [ + "6", + "7", + "Olives - processing", + null, + null, + null + ], + [ + "7", + "8", + "Corn - all", + null, + null, + null + ], + [ + "8", + "9", + "Hay - alfalfa", + null, + null, + null + ], + [ + "9", + "10", + "Silage - wheat", + null, + null, + null + ], + [ + "10", + "11", + "Rice", + null, + null, + null + ], + [ + "11", + "12", + "Peaches", + null, + null, + null + ], + [ + "12", + "13", + "Potatoes", + null, + null, + null + ], + [ + "13", + "14", + "Cucumbers and pickles", + null, + null, + null + ], + [ + "14", + "15", + "Pistachios", + null, + null, + null + ], + [ + "15", + "16", + "Cotton", + null, + null, + null + ], + [ + "16", + "17", + "Olives - market", + null, + null, + null + ], + [ + "17", + "18", + "", + null, + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 18 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idnamenotedescriptionuri
01Tomatoes for processingNoneNoneNone
12GrapesNoneNoneNone
23AlmondsNoneNoneNone
34WalnutsNoneNoneNone
45Sweet potatoesNoneNoneNone
56AlgaeNoneNoneNone
67Olives - processingNoneNoneNone
78Corn - allNoneNoneNone
89Hay - alfalfaNoneNoneNone
910Silage - wheatNoneNoneNone
1011RiceNoneNoneNone
1112PeachesNoneNoneNone
1213PotatoesNoneNoneNone
1314Cucumbers and picklesNoneNoneNone
1415PistachiosNoneNoneNone
1516CottonNoneNoneNone
1617Olives - marketNoneNoneNone
1718NoneNoneNone
\n", + "
" + ], + "text/plain": [ + " id name note description uri\n", + "0 1 Tomatoes for processing None None None\n", + "1 2 Grapes None None None\n", + "2 3 Almonds None None None\n", + "3 4 Walnuts None None None\n", + "4 5 Sweet potatoes None None None\n", + "5 6 Algae None None None\n", + "6 7 Olives - processing None None None\n", + "7 8 Corn - all None None None\n", + "8 9 Hay - alfalfa None None None\n", + "9 10 Silage - wheat None None None\n", + "10 11 Rice None None None\n", + "11 12 Peaches None None None\n", + "12 13 Potatoes None None None\n", + "13 14 Cucumbers and pickles None None None\n", + "14 15 Pistachios None None None\n", + "15 16 Cotton None None None\n", + "16 17 Olives - market None None None\n", + "17 18 None None None" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "summary_stats = df2.groupby(['Resource', 'Parameter'])['Value'].agg(['mean', 'median', 'min', 'max', 'std', 'count'])" + "from sqlmodel import Session, select, create_engine\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "\n", + "# Database Connection\n", + "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", + "engine = create_engine(DATABASE_URL)\n", + "print(f\"Connected to database.\")\n", + "\n", + "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", + "\n", + "#reorders columns so id and name are first\n", + "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", + "\n", + "primary_ag_product = primary_ag_product[[*cols]]\n", + "\n", + "primary_ag_product\n" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "52090a35-9379-4dc9-b918-583b23513ddd", + "rows": [ + [ + "0", + null, + "1", + "Tomatoes for processing", + null, + null + ], + [ + "1", + null, + "2", + "Grapes", + null, + null + ], + [ + "2", + null, + "3", + "Almonds", + null, + null + ], + [ + "3", + null, + "4", + "Walnuts", + null, + null + ], + [ + "4", + null, + "5", + "Sweet potatoes", + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
descriptionidnamenoteuri
0None1Tomatoes for processingNoneNone
1None2GrapesNoneNone
2None3AlmondsNoneNone
3None4WalnutsNoneNone
4None5Sweet potatoesNoneNone
\n", + "
" + ], + "text/plain": [ + " description id name note uri\n", + "0 None 1 Tomatoes for processing None None\n", + "1 None 2 Grapes None None\n", + "2 None 3 Almonds None None\n", + "3 None 4 Walnuts None None\n", + "4 None 5 Sweet potatoes None None" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "# --- project root discovery (unchanged) ---\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# --- imports ---\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", + "\n", + "# --- query + dataframe ---\n", + "with Session(engine) as db:\n", + " stmt = select(*PrimaryAgProduct.__table__.columns)\n", + " rows = db.execute(stmt).mappings().all()\n", + "\n", + "df = pd.DataFrame(rows)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
21:07:54.222 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "21:07:54.222 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
21:07:59.598 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "21:07:59.598 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
21:07:59.601 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "21:07:59.601 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", + " replace_name_with_id_df,\n", + ") \n", + "\n", + "#This extractst the raw proximate data\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", + "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", + "\n", + "#this replaces the names with IDs\n", + "with Session(engine) as db:\n", + " parameter_ids = replace_name_with_id_df(\n", + " db=db,\n", + " df=test_df,\n", + " ref_model=Parameter,\n", + " name_column_name=\"name\", # column in df + table\n", + " id_column_name=\"id\", # PK column in table\n", + " final_column_name=\"parameter_id\"\n", + " )\n", + "\n", + "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", + "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", + "\n", + "#resource_id_mapping\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "string", + "type": "string" + }, + { + "name": "record_id", + "rawType": "string", + "type": "string" + }, + { + "name": "prepared_sample", + "rawType": "string", + "type": "string" + }, + { + "name": "resource", + "rawType": "string", + "type": "string" + }, + { + "name": "preparation_method", + "rawType": "string", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "string", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "string", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "Int32", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "string", + "type": "string" + }, + { + "name": "value", + "rawType": "Float32", + "type": "float" + }, + { + "name": "unit", + "rawType": "string", + "type": "string" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "string", + "type": "string" + }, + { + "name": "note", + "rawType": "string", + "type": "string" + }, + { + "name": "analysis_type", + "rawType": "string", + "type": "string" + }, + { + "name": "equipment", + "rawType": "string", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "string", + "type": "string" + }, + { + "name": "analyst_email", + "rawType": "string", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "string", + "type": "string" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "91fad99c-9137-48ce-af0f-4004b3241e88", + "rows": [ + [ + "0", + "3EE2993D-86E3-1F16-C7EA-F8D555E114", + "(85)E114", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "14.16", + "% dry weight", + "2025-01-23 09:00:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "1", + "46878EF9-1226-22A0-D5D8-CF65E241CB", + "(85)41CB", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "14.18", + "% dry weight", + "2025-01-23 09:00:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "2", + "76A7A2F4-C4E4-E60F-1187-DEC6E02246", + "(85)2246", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "14.12", + "% dry weight", + "2025-01-23 09:00:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "3", + "7A136832-286B-07CB-62DE-ACF52F9311", + "(85)9311", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "15.74", + "% dry weight", + "2025-01-23 09:00:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "4", + "B709ECEE-F9A6-A55D-A59E-93B7B863D7", + "(85)63D7", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "15.75", + "% dry weight", + "2025-01-23 09:01:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "5", + "35080403-BB4B-776B-E045-4627C307BD", + "(85)07BD", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "15.68", + "% dry weight", + "2025-01-23 09:01:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "6", + "D83F3328-931D-582E-B563-4DFF118C05", + "(85)8C05", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "9.88", + "% dry weight", + "2025-01-23 09:01:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "7", + "B3F6AA18-E51A-2585-8417-113DE0F026", + "(85)F026", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "9.25", + "% dry weight", + "2025-01-23 09:01:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "8", + "094CFB14-0D19-C54E-51C4-6BA548A6D4", + "(85)A6D4", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "9.43", + "% dry weight", + "2025-01-23 09:02:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "9", + "99D17377-62B8-2124-5391-62BC1F4630", + "(85)4630", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "11.22", + "% dry weight", + "2025-01-23 09:02:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "10", + "9B784971-B92C-A870-DA69-C4D771B716", + "(85)B716", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "10.51", + "% dry weight", + "2025-01-23 09:02:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "11", + "57C508A2-7489-2AFB-23B2-1630C38BD5", + "(85)8BD5", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "10.72", + "% dry weight", + "2025-01-23 09:02:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "12", + "40EAE6DB-4CD6-3574-949F-70B03A51C8", + "(85)51C8", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "39.54", + "% dry weight", + "2025-01-23 09:03:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "13", + "A752F059-ECF4-FA5B-A9D2-7DF8963329", + "(85)3329", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "39.9", + "% dry weight", + "2025-01-23 09:03:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "14", + "BEB4369F-D40F-DF6A-30F8-75B91C5B1B", + "(85)5B1B", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "40.88", + "% dry weight", + "2025-01-23 09:03:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "15", + "00CAC475-8E18-2318-AA6A-2AE068CADF", + "(D2)CADF", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "19.04", + "% dry weight", + "2025-01-23 09:03:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "16", + "2DA01B52-D301-53C9-220D-A51216BDB6", + "(D2)BDB6", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "18.9", + "% dry weight", + "2025-01-23 09:04:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "17", + "F8D1C324-F3F5-7754-494E-7BBFB262D0", + "(D2)62D0", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "18.98", + "% dry weight", + "2025-01-23 09:04:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "18", + "84FE8C5C-5D37-B0EA-6488-B883CEA7F3", + "(D2)A7F3", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "21.15", + "% dry weight", + "2025-01-23 09:04:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "19", + "E705EAF1-69EB-ADB3-620B-1DD93F4D07", + "(D2)4D07", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "21.0", + "% dry weight", + "2025-01-23 09:04:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "20", + "F6A97D04-E898-06F1-A90B-0EE254D758", + "(D2)D758", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "21.08", + "% dry weight", + "2025-01-23 09:05:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "21", + "6948E00A-4858-6F19-23BC-AA016F12CB", + "(D2)12CB", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "8.84", + "% dry weight", + "2025-01-23 09:05:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "22", + "EEC9EA54-3C1A-6154-B953-401C89CEA7", + "(D2)CEA7", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "8.46", + "% dry weight", + "2025-01-23 09:05:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "23", + "62E4A908-ED27-590F-0DD2-EA254F1E4C", + "(D2)1E4C", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "8.72", + "% dry weight", + "2025-01-23 09:05:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "24", + "AA2BBE0D-FD98-98F9-9880-CDB56142A2", + "(D2)42A2", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "10.04", + "% dry weight", + "2025-01-23 09:06:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "25", + "5186F84F-8BBB-98A7-8FF9-CF85CCD344", + "(D2)D344", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "9.62", + "% dry weight", + "2025-01-23 09:06:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "26", + "2BAF4360-89A7-C605-2141-827901DA56", + "(D2)DA56", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "9.91", + "% dry weight", + "2025-01-23 09:06:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "27", + "B9D049D7-363C-365F-3750-331CEC2FAE", + "(D2)2FAE", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "30.53", + "% dry weight", + "2025-01-23 09:06:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "28", + "4E056928-76F1-99E1-4F78-FD31CC0B71", + "(D2)0B71", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "31.57", + "% dry weight", + "2025-01-23 09:07:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "29", + "4734CD60-9A40-8ECD-7E0D-FA2AD6D96D", + "(D2)D96D", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "31.95", + "% dry weight", + "2025-01-23 09:07:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "30", + "2CD57693-9038-EC9E-EC61-E91793A602", + "(28)A602", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "11.71", + "% dry weight", + "2025-01-23 09:07:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "31", + "BD14399E-2A3B-56A9-AC9C-1689DA75A7", + "(28)75A7", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "13.66", + "% dry weight", + "2025-01-23 09:07:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "32", + "6635AA49-3247-3A08-D00F-CA5447491E", + "(28)491E", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "13.84", + "% dry weight", + "2025-01-23 09:08:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "33", + "EE7D6738-BEE5-87A1-6387-89AE03CFC4", + "(28)CFC4", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "13.01", + "% dry weight", + "2025-01-23 09:08:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "34", + "ACD2BFD6-EA7C-8CBB-F4DA-CAA860F6C9", + "(28)F6C9", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "15.17", + "% dry weight", + "2025-01-23 09:08:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "35", + "1BFB37F6-A5D7-E88E-84A4-DBA38059B5", + "(28)59B5", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "15.37", + "% dry weight", + "2025-01-23 09:08:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "36", + "D6CD55E5-19B7-5E3A-EEE6-95564F8C33", + "(28)8C33", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "9.36", + "% dry weight", + "2025-01-23 09:09:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "37", + "17DE0AED-2CC5-2B5A-E521-049F78F0FE", + "(28)F0FE", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "11.26", + "% dry weight", + "2025-01-23 09:09:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "38", + "4D593E32-1FDA-373B-EE45-38FB01847B", + "(28)847B", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "11.06", + "% dry weight", + "2025-01-23 09:09:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "39", + "84904964-58B9-A45E-3F79-6747BF7919", + "(28)7919", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "10.64", + "% dry weight", + "2025-01-23 09:09:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "40", + "808F6FAC-F9A6-A9EB-56D5-EECA3BA3D2", + "(28)A3D2", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "12.8", + "% dry weight", + "2025-01-23 09:10:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "41", + "67FD9F7C-8CE8-04F4-2CB5-6C42532429", + "(28)2429", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "12.56", + "% dry weight", + "2025-01-23 09:10:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "42", + "90E8067F-045F-3E6E-5BAA-94BEE9D0CF", + "(28)D0CF", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "40.12", + "% dry weight", + "2025-01-23 09:10:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "43", + "112546A5-714C-B29F-808B-CBDE6DE48E", + "(28)E48E", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "39.76", + "% dry weight", + "2025-01-23 09:10:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "44", + "2FF25721-ED5B-05E4-2DC5-5799261266", + "(28)1266", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "40.34", + "% dry weight", + "2025-01-23 09:11:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "45", + "7BC7134E-F8BB-F153-DC50-67A1161509", + "(AB)1509", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(AB)1", + "7.95", + "% dry weight", + "2025-01-22 09:00:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "46", + "F442FF50-46CF-317D-9708-14F6461273", + "(AB)1273", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(AB)2", + "7.77", + "% dry weight", + "2025-01-22 09:00:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "47", + "C80175E1-8D21-920C-B982-A246941FCB", + "(AB)1FCB", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(AB)3", + "7.96", + "% dry weight", + "2025-01-22 09:00:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "48", + "63CF426B-99C5-3896-C472-49BBAE19CD", + "(AB)19CD", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(AB)1", + "8.84", + "% dry weight", + "2025-01-22 09:00:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "49", + "A9D902B0-033D-29B4-C477-1A7D3E57F4", + "(AB)57F4", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(AB)2", + "8.64", + "% dry weight", + "2025-01-22 09:01:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ] + ], + "shape": { + "columns": 21, + "rows": 390 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cmp_uuid_033record_idprepared_sampleresourcepreparation_methodstorage_condexper_abbrevrepl_norepl_idvalue...created_atupdated_atqc_resultnoteanalysis_typeequipmentraw_data_urlanalyst_emailupload_statusparameter_id
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)114.16...2025-01-23 09:00:01NaTpassChemical compositionxkang2@lbl.govready6
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)214.18...2025-01-23 09:00:16NaTpassChemical compositionxkang2@lbl.govready6
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)314.12...2025-01-23 09:00:31NaTpassChemical compositionxkang2@lbl.govready6
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)115.74...2025-01-23 09:00:46NaTpassChemical compositionxkang2@lbl.govready4
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)215.75...2025-01-23 09:01:01NaTpassChemical compositionxkang2@lbl.govready4
..................................................................
38579799601-0E25-3832-3BBF-F4BD9EE973(A0)E973Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)217.290001...2025-03-04 09:55:00NaTpassChemical compositionxkang2@lbl.govnot ready5
386A4E3393F-72C3-A0ED-C041-105A33031A(A0)031AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)317.5...2025-03-04 09:55:15NaTpassChemical compositionxkang2@lbl.govnot ready5
3873D5A85F8-31A2-0CC2-74E6-05D8197C3A(A0)7C3AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1Cmp17xk(A0)135.970001...2025-03-04 09:55:30NaTpassChemical compositionxkang2@lbl.govnot ready3
388C7AE456E-4DCD-AE34-A0D0-BD39D04E42(A0)4E42Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)235.279999...2025-03-04 09:55:45NaTpassChemical compositionxkang2@lbl.govnot ready3
389A5E617A2-9831-3E9F-7072-6C5FB8129B(A0)129BHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)334.990002...2025-03-04 09:56:00NaTpassChemical compositionxkang2@lbl.govnot ready3
\n", + "

390 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " cmp_uuid_033 record_id prepared_sample \\\n", + "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", + "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", + "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", + "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", + "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", + ".. ... ... ... \n", + "385 79799601-0E25-3832-3BBF-F4BD9EE973 (A0)E973 Hum-AlmBr024KM2(A0) \n", + "386 A4E3393F-72C3-A0ED-C041-105A33031A (A0)031A Hum-AlmBr024KM2(A0) \n", + "387 3D5A85F8-31A2-0CC2-74E6-05D8197C3A (A0)7C3A Hum-AlmBr024KM2(A0) \n", + "388 C7AE456E-4DCD-AE34-A0D0-BD39D04E42 (A0)4E42 Hum-AlmBr024KM2(A0) \n", + "389 A5E617A2-9831-3E9F-7072-6C5FB8129B (A0)129B Hum-AlmBr024KM2(A0) \n", + "\n", + " resource preparation_method storage_cond exper_abbrev \\\n", + "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + ".. ... ... ... ... \n", + "385 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "386 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "387 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "388 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "389 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "\n", + " repl_no repl_id value ... created_at updated_at \\\n", + "0 1 Cmp04xk(85)1 14.16 ... 2025-01-23 09:00:01 NaT \n", + "1 2 Cmp04xk(85)2 14.18 ... 2025-01-23 09:00:16 NaT \n", + "2 3 Cmp04xk(85)3 14.12 ... 2025-01-23 09:00:31 NaT \n", + "3 1 Cmp04xk(85)1 15.74 ... 2025-01-23 09:00:46 NaT \n", + "4 2 Cmp04xk(85)2 15.75 ... 2025-01-23 09:01:01 NaT \n", + ".. ... ... ... ... ... ... \n", + "385 2 Cmp17xk(A0)2 17.290001 ... 2025-03-04 09:55:00 NaT \n", + "386 3 Cmp17xk(A0)3 17.5 ... 2025-03-04 09:55:15 NaT \n", + "387 1 Cmp17xk(A0)1 35.970001 ... 2025-03-04 09:55:30 NaT \n", + "388 2 Cmp17xk(A0)2 35.279999 ... 2025-03-04 09:55:45 NaT \n", + "389 3 Cmp17xk(A0)3 34.990002 ... 2025-03-04 09:56:00 NaT \n", + "\n", + " qc_result note analysis_type equipment raw_data_url \\\n", + "0 pass Chemical composition \n", + "1 pass Chemical composition \n", + "2 pass Chemical composition \n", + "3 pass Chemical composition \n", + "4 pass Chemical composition \n", + ".. ... ... ... ... ... \n", + "385 pass Chemical composition \n", + "386 pass Chemical composition \n", + "387 pass Chemical composition \n", + "388 pass Chemical composition \n", + "389 pass Chemical composition \n", + "\n", + " analyst_email upload_status parameter_id \n", + "0 xkang2@lbl.gov ready 6 \n", + "1 xkang2@lbl.gov ready 6 \n", + "2 xkang2@lbl.gov ready 6 \n", + "3 xkang2@lbl.gov ready 4 \n", + "4 xkang2@lbl.gov ready 4 \n", + ".. ... ... ... \n", + "385 xkang2@lbl.gov not ready 5 \n", + "386 xkang2@lbl.gov not ready 5 \n", + "387 xkang2@lbl.gov not ready 3 \n", + "388 xkang2@lbl.gov not ready 3 \n", + "389 xkang2@lbl.gov not ready 3 \n", + "\n", + "[390 rows x 21 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parameter_ids\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "summary_stats" + "import numpy as np\n", + "from prefect import get_run_logger\n", + "\n", + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "resource['name'] = resource['name'].str.lower()\n", + "\n", + "resource['name'] = resource['name'].replace('', np.nan)\n", + "resource.dropna(subset=['name'], inplace=True)\n", + "\n", + "\n", + "resource\n" ] }, { @@ -152,7 +4167,12 @@ "metadata": {}, "outputs": [], "source": [ - "\n" + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "#this converts the entire dataframe to lowercase\n", + "df = df.map(lambda x: x.lower() if isinstance(x, str) else x)\n", + "\n", + "print(df)" ] }, { @@ -160,18 +4180,32 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "field_sample = pd.read_sql_query(\"SELECT * FROM field_sample\", con=engine)\n", + "\n", + "field_sample = field_sample[['id', 'name', 'resource_id']]\n", + "\n", + "field_sample" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "default", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.12" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py new file mode 100644 index 00000000..6c92cb4f --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -0,0 +1,81 @@ +from typing import Type, TypeVar, Any + +import pandas as pd +from sqlalchemy.orm import Session +from sqlalchemy import select + +ModelType = TypeVar("ModelType", bound=Any) + +def replace_id_with_name_df( + db: Session, + df: pd.DataFrame, + ref_model: Type[ModelType], + id_column_name: str, + name_column_name: str, +) -> pd.DataFrame: + # Fetch reference table rows as mappings + rows = db.execute( + select(*ref_model.__table__.columns) + ).mappings().all() + + # Build ID → name map + id_to_name_map = { + row[id_column_name]: row[name_column_name] + for row in rows + } + + df_copy = df.copy() + df_copy[name_column_name] = df_copy[id_column_name].map(id_to_name_map) + df_copy = df_copy.drop(columns=[id_column_name]) + + return df_copy + + +def replace_name_with_id_df( + db: Session, + df: pd.DataFrame, + ref_model: Type[ModelType], + name_column_name: str, + id_column_name: str, + final_column_name: str +) -> pd.DataFrame: + # Fetch reference table rows + rows = db.execute( + select(*ref_model.__table__.columns) + ).mappings().all() + + name_to_id_map = { + row[name_column_name]: row[id_column_name] + for row in rows + } + + df_copy = df.copy() + + unique_names = set(df_copy[name_column_name].dropna().unique()) + new_names = unique_names - set(name_to_id_map.keys()) + + if new_names: + for name in new_names: + new_record = ref_model(**{name_column_name: name}) + db.add(new_record) + + # Commit once, not per row + db.commit() + + # Refresh and update lookup + refreshed = db.execute( + select(ref_model).where( + getattr(ref_model, name_column_name).in_(new_names) + ) + ).scalars().all() + + for record in refreshed: + name_to_id_map[getattr(record, name_column_name)] = getattr( + record, id_column_name + ) + + df_copy[id_column_name] = df_copy[name_column_name].map(name_to_id_map) + df_copy = df_copy.drop(columns=[name_column_name]) + df_copy = df_copy.rename(columns={id_column_name: final_column_name}) + + return df_copy diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql deleted file mode 100644 index c0d3337d..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql +++ /dev/null @@ -1,15 +0,0 @@ --- This script inserts 10 sample rows into the biomass table. --- Note: This assumes that valid records exist in the primary_product, taxonomy, and biomass_type tables --- for the foreign key IDs used below. - -INSERT INTO biomass (biomass_name, primary_product_id, taxonomy_id, biomass_type_id, biomass_notes) VALUES -('Corn Stover', 1, 101, 1, 'Lignocellulosic biomass from corn stalks and leaves.'), -('Switchgrass', 2, 102, 1, 'Perennial grass used as a dedicated energy crop.'), -('Pine Wood Chips', 3, 103, 2, 'Woody biomass from pine trees, chipped for processing.'), -('Algae Culture', 4, 104, 3, 'Aquatic biomass grown for biofuel production.'), -('Municipal Solid Waste (MSW)', 5, 105, 4, 'Organic fraction of household waste.'), -('Food Waste', 6, 106, 4, 'Post-consumer food scraps from restaurants.'), -('Manure Slurry', 7, 107, 5, 'Animal waste from dairy farms, used in anaerobic digestion.'), -('Miscanthus Giganteus', 2, 108, 1, 'High-yield sterile hybrid grass for bioenergy.'), -('Willow Coppice', 3, 109, 2, 'Short-rotation coppice willow for biomass energy.'), -('Sugarcane Bagasse', 1, 110, 1, 'Fibrous residue left after sugarcane crushing.'); From 5e961aa19d87a1df708b83051e01a15efeb1fd8c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Sat, 3 Jan 2026 08:39:43 -0700 Subject: [PATCH 5/7] modifying name_id_swap_function --- .../pipeline/utils/name_id_swap.py | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py index 6c92cb4f..d9290968 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -31,51 +31,60 @@ def replace_id_with_name_df( return df_copy + def replace_name_with_id_df( db: Session, df: pd.DataFrame, - ref_model: Type[ModelType], - name_column_name: str, + ref_model, + df_name_column: str, + model_name_attr: str, id_column_name: str, - final_column_name: str + final_column_name: str, ) -> pd.DataFrame: - # Fetch reference table rows + """ + Replace a DataFrame name column with foreign key IDs from a SQLAlchemy table. + Creates missing reference records if needed. + """ + + # 1. Fetch existing reference rows (name + id only) rows = db.execute( - select(*ref_model.__table__.columns) - ).mappings().all() + select( + getattr(ref_model, model_name_attr), + getattr(ref_model, id_column_name), + ) + ).all() - name_to_id_map = { - row[name_column_name]: row[id_column_name] - for row in rows - } + name_to_id_map = {name: id_ for name, id_ in rows} df_copy = df.copy() - unique_names = set(df_copy[name_column_name].dropna().unique()) + # 2. Determine which names are new + unique_names = set(df_copy[df_name_column].dropna().unique()) new_names = unique_names - set(name_to_id_map.keys()) + # 3. Insert missing reference rows if new_names: for name in new_names: - new_record = ref_model(**{name_column_name: name}) + new_record = ref_model(**{model_name_attr: name}) db.add(new_record) - # Commit once, not per row - db.commit() + # Flush to get IDs without ending the transaction + db.flush() - # Refresh and update lookup + # Re-query just-created rows refreshed = db.execute( select(ref_model).where( - getattr(ref_model, name_column_name).in_(new_names) + getattr(ref_model, model_name_attr).in_(new_names) ) ).scalars().all() for record in refreshed: - name_to_id_map[getattr(record, name_column_name)] = getattr( - record, id_column_name - ) + name_to_id_map[ + getattr(record, model_name_attr) + ] = getattr(record, id_column_name) - df_copy[id_column_name] = df_copy[name_column_name].map(name_to_id_map) - df_copy = df_copy.drop(columns=[name_column_name]) - df_copy = df_copy.rename(columns={id_column_name: final_column_name}) + # 4. Replace name column with ID column + df_copy[final_column_name] = df_copy[df_name_column].map(name_to_id_map) + df_copy = df_copy.drop(columns=[df_name_column]) return df_copy From 6d7990b252f7f86b948946ce958cc48bbb830f2c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Sun, 4 Jan 2026 20:06:00 -0700 Subject: [PATCH 6/7] was trying to fix dev container kernel issue. Did not succeed. Also made new ETL notebook and modified some of the gsheet extraction notebook --- pixi.lock | 1606 +++++- pixi.toml | 1 + .../pipeline/utils/etl_notebook.ipynb | 1787 ++++++ .../utils/gsheet_extraction_notebook.ipynb | 4929 ++++------------- .../pipeline/utils/name_id_swap.py | 8 +- 5 files changed, 4407 insertions(+), 3924 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb diff --git a/pixi.lock b/pixi.lock index faeb4843..5581bcfd 100644 --- a/pixi.lock +++ b/pixi.lock @@ -10,13 +10,25 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/argon2-cffi-bindings-25.1.0-py312h4c3975b_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/asyncpg-0.29.0-py312h66e93f0_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bcrypt-5.0.0-py312h868fb18_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.2.0-py312hdb49522_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py312h460c074_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -28,6 +40,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/cryptography-46.0.3-py312ha4b625e_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.18-py312h8285ef7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/docker-cli-28.3.1-h73b1eb8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/docker-compose-2.39.2-h280c20c_0.conda @@ -35,22 +48,46 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda @@ -67,12 +104,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.2-he9a06e4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.3-py312h8a5da7c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.4-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -81,7 +127,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.1.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda @@ -94,30 +142,49 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.12-hd63d673_1_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/pyyaml-6.0.3-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.1.0-py312hfb55c3c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.30.0-py312h868fb18_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.3-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py312hd9148b4_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/wrapt-2.0.1-py312h4c3975b_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h387f397_9.conda @@ -129,12 +196,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -147,7 +210,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -156,29 +218,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -195,7 +248,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -210,19 +262,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/14/9a39b7c9e007968411bc3c843cc14cf15437510c0a9991f080cab654fd16/regex-2025.10.23-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3e/cd/49ce51767b879cde77e7ad9fae164ea15dce3616fe591d9ea1df51152706/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/61/62/150c841f24cda9e30f588ef396ed83f64cfdc13b92d2f925bb96df337ba9/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -230,7 +277,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -249,13 +295,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/59/61d8e9f1734069049abe9e593961de602397c7194712346906c075fec65f/uv-0.9.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels @@ -264,13 +307,25 @@ environments: linux-aarch64: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/argon2-cffi-bindings-25.1.0-py312hcd1a082_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/asyncpg-0.29.0-py312hb2c0f52_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bcrypt-5.0.0-py312h5eb8f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-python-1.1.0-py312h1ab2c47_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py312h2fc7fbd_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -282,6 +337,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cryptography-46.0.3-py312h4cd2d69_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/debugpy-1.8.17-py312hf55c4e8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/docker-cli-28.3.1-h06eaf92_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/docker-compose-2.39.2-h80f16a2_0.conda @@ -289,22 +345,46 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-h9df1782_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda @@ -321,12 +401,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/markupsafe-3.0.3-py312hd077ced_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -335,7 +424,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/psutil-7.1.2-py312hd41f8a7_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda @@ -348,30 +439,49 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.12-hcfbf8c2_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pyyaml-6.0.3-py312ha4530ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pyzmq-27.1.0-py312h4552c38_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rpds-py-0.30.0-py312h75d7d99_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tornado-6.5.2-py312hefbd42c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ukkonen-1.0.1-py312h451a7dd_5.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wrapt-1.17.3-py312hcd1a082_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zeromq-4.3.5-hefbcea8_9.conda @@ -383,12 +493,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -401,7 +507,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -410,29 +515,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -449,7 +545,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -464,19 +559,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/95/888f069c89e7729732a6d7cca37f76b44bfb53a1e35dda8a2c7b65c1b992/regex-2025.10.23-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/ea/42d243d3a586beb72c77fa5def0487daf827210069a95f36328e869599ea/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/c5/346c7094344a60419764b4b1334d9e0285031c961176ff88ffb652405b0c/ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux2014_aarch64.whl @@ -484,7 +574,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -503,13 +592,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/ac/090dbde63abb56001190392d29ca2aa654eebc146a693b5dda68da0df2fb/uv-0.9.7-py3-none-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels @@ -517,14 +603,26 @@ environments: - pypi: ./src/ca_biositing/webservice osx-64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/argon2-cffi-bindings-25.1.0-py313hf050af9_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/asyncpg-0.29.0-py313ha37c0e0_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/bcrypt-5.0.0-py313hcc225dc_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/brotli-python-1.2.0-py313hd4eab94_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cffi-2.0.0-py313hf57695f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -536,6 +634,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/cryptography-46.0.3-py313h0218d6d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/debugpy-1.8.17-py313hff8d55d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/docker-cli-28.3.1-h2287256_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/docker-compose-2.39.2-h5282f48_0.conda @@ -543,21 +642,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.21.3-h37d8d59_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.4-h3d58e20_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libedit-3.1.20250104-pl5321ha958ccf_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.1-h21dd04a_0.conda @@ -567,12 +690,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libsodium-1.0.20-hfdf4475_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.50.4-h39a8b3b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/markupsafe-3.0.3-py313h0f4d31d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.5.4-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -581,42 +713,65 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/psutil-7.1.3-py313hcb05632_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pynacl-1.6.0-py313hf050af9_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-core-12.1-py313h07bcf3a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-framework-cocoa-12.1-py313hf669bc3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-8.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.0.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.13.9-h17c18a5_101_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.9-h4df99d1_101.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py313h0f4d31d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyzmq-27.1.0-py312hb7d603e_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h7cca4af_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/rpds-py-0.30.0-py313hcc225dc_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tornado-6.5.2-py313h585f44e_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ukkonen-1.0.1-py313hc551f4f_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/wrapt-2.0.0-py313hf050af9_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zeromq-4.3.5-h6c33b1e_9.conda @@ -628,12 +783,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -646,7 +797,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -655,29 +805,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -693,7 +834,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -708,19 +848,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4c/93/181070cd1aa2fa541ff2d3afcf763ceecd4937b34c615fa92765020a6c90/regex-2025.10.23-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d3/03/ce566d92611dfac0085c2f4b048cd53ed7c274a5c05974b882a908d540a2/rpds_py-0.28.0-cp313-cp313-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d7/ae/e3811f05415594025e96000349d3400978adaed88d8f98d494352d9761ee/ruamel.yaml.clib-0.2.14-cp313-cp313-macosx_10_13_universal2.whl @@ -728,7 +863,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -747,13 +881,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6f/b7/1b1ff8dfde05e9d27abf29ebf22da48428fe1e16f0b4d65a839bd2211303/uv-0.9.7-py3-none-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b4/01/543ce9ecaa8dd288b5bd830db221a1508b3662196022f001818f4de36ef3/whenever-0.9.3-cp313-cp313-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl @@ -762,14 +893,26 @@ environments: - pypi: ./src/ca_biositing/webservice osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/argon2-cffi-bindings-25.1.0-py313h6535dbc_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/asyncpg-0.29.0-py313h20a7fcf_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bcrypt-5.0.0-py313h2c089d5_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.2.0-py313hde1f3bb_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cffi-2.0.0-py313h224173a_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -781,6 +924,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-46.0.3-py313h76c770c_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.19-py313hc37fe24_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/docker-cli-28.3.1-hbc156a2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/docker-compose-2.39.2-h925e9cb_0.conda @@ -788,21 +932,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.21.3-h237132a_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321hafb1f1b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda @@ -812,12 +980,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsodium-1.0.20-h99b78c6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py313h7d74516_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -826,42 +1003,65 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.1.3-py313h9734d34_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pynacl-1.6.1-py313h6535dbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-core-12.1-py313h40b429f_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-framework-cocoa-12.1-py313hcc5defa_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-8.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.0.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.11-hfc2f54d_100_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/pyyaml-6.0.3-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-27.1.0-py312hd65ceae_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/rpds-py-0.30.0-py313h2c089d5_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tornado-6.5.4-py313h6535dbc_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ukkonen-1.0.1-py313hc50a443_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.0.1-py313h6535dbc_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h888dc83_9.conda @@ -873,12 +1073,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -891,7 +1087,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -900,29 +1095,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -938,7 +1124,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -953,19 +1138,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/c5/9d37fbe3a40ed8dda78c23e1263002497540c0d1522ed75482ef6c2000f0/regex-2025.10.23-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/00/34/1c61da1b25592b86fd285bd7bd8422f4c9d748a7373b46126f9ae792a004/rpds_py-0.28.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d6/03/a1baa5b94f71383913f21b96172fb3a2eb5576a4637729adbf7cd9f797f8/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_11_0_arm64.whl @@ -973,7 +1153,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -992,13 +1171,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f5/7d/b618174d8a8216af350398ace03805b2b2df6267b1745abf45556c2fda58/uv-0.9.7-py3-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7b/d7/d89a6f1b1ea4a2768c813443d7be007377402fa0db99c62c393965165159/whenever-0.9.3-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl @@ -1007,14 +1183,26 @@ environments: - pypi: ./src/ca_biositing/webservice win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/argon2-cffi-bindings-25.1.0-py313h5ea7bf4_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/asyncpg-0.29.0-py313ha7868ed_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/backports.zstd-1.2.0-py313h2a31948_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bcrypt-5.0.0-py313hfbe8231_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/brotli-python-1.2.0-py313h3ebfc14_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.11.12-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.5.0-pyhd8ed1ab_0.conda @@ -1026,6 +1214,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/cryptography-46.0.3-py313hf5c5e30_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.19-py313h927ade5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/docker-cli-28.3.1-h6a83c73_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/docker-compose-2.40.1-h6a83c73_0.conda @@ -1033,20 +1222,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh6dadd2b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.8.0-pyhe2676ad_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.7.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyh6dadd2b_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda @@ -1054,18 +1268,29 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libsodium-1.0.20-hc70643c_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/markupsafe-3.0.3-py313hd650c13_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pip-25.3-pyh145f28c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhf9edf01_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.5.1-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/psutil-7.1.3-py313h5fd188c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda @@ -1077,49 +1302,67 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.13.11-h09917c8_100_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pywin32-311-py313h40c08fc_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh07e9846_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/win-64/pywinpty-2.0.15-py313h5813708_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py313hd650c13_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyzmq-27.1.0-py312hbb5da91_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/rpds-py-0.30.0-py313hfbe8231_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh6dadd2b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh6dadd2b_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.3-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tornado-6.5.4-py313h5ea7bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ukkonen-1.0.1-py313hf069bd2_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.6.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/win_inet_pton-1.1.0-pyh7428d3b_8.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/winpty-0.4.3-4.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/wrapt-2.0.1-py313h5ea7bf4_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zeromq-4.3.5-h5bddc39_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.23.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ef/39/b2181148075272edfbbd6d87e6cd78cc71dca243446fa3b381fd4116950b/aiosqlite-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/df/343d125241f8cd3c9af58fd09688cf2bf59cc1edfd609adafef3556ce8ec/apprise-1.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/fc/1d7b80d0eb7b714984ce40efc78859c022cd930e402f599d8ca9e39c78a4/cachetools-6.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz @@ -1134,7 +1377,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/78/a850fed8aeef96d4a99043c90b818b2ed5419cd5b24a4049fd7cfb9f1471/fakeredis-2.33.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -1143,31 +1385,21 @@ environments: - pypi: https://files.pythonhosted.org/packages/9c/83/3b1d03d36f224edded98e9affd0467630fc09d766c0e56fb1498cbb04a9b/griffe-1.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/7b/bca5613a0c3b542420cf92bd5e5fb8ebd5435ce1011a091f66bb7693285e/humanize-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/b4/55e885834c847ea610e111d87b9ed4768f0afdaeebc00cd46810f25029f6/lupa-2.6-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -1186,7 +1418,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/59/3d/68243e3e219d765a3012a1e396b3ea8566c3ea2b60283e27327c0f4415e9/prefect-3.6.7-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/99/10/72f6f213b8f0bce36eff21fda0a13271834e9eeff7f9609b01afdc253c79/py_key_value_aio-0.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/e4/b8b0a03ece72f47dce2307d36e1c34725b7223d209fc679315ffe6a4e2c3/py_key_value_shared-0.3.0-py3-none-any.whl @@ -1203,22 +1434,16 @@ environments: - pypi: https://files.pythonhosted.org/packages/9b/48/efb1b1d3f3aee8cfc9f256738ca6e79ec362edbfc3a3abecbaf84db04643/PyShEx-0.8.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - - pypi: https://files.pythonhosted.org/packages/51/e5/fecf13f06e5e5f67e8837d777d1bc43fac0ed2b77a676804df5c34744727/python_json_logger-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/f0/8956f8a86b20d7bb9d6ac0187cf4cd54d8065bc9a1a09eb8011d4d326596/redis-7.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8b/00/6e29bb314e271a743170e53649db0fdb8e8ff0b64b4f425f5602f4eb9014/regex-2025.11.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/af/fe/b6045c782f1fd1ae317d2a6ca1884857ce5c20f59befe6ab25a8603c43a7/ruamel_yaml-0.18.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/06/c4/c124fbcef0684fcf3c9b72374c2a8c35c94464d8694c50f37eef27f5a145/ruamel_yaml_clib-0.2.15-cp313-cp313-win_amd64.whl @@ -1226,7 +1451,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl @@ -1246,16 +1470,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/9f/e7/84381106a701c812652405f4d74093f6da7c2748a639084fd2c092cd2c78/whenever-0.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d5/e4/62a677feefde05b12a70a4fc9bdc8558010182a801fbcab68cb56c2b0986/xarray-2025.12.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels - pypi: ./src/ca_biositing/pipeline - pypi: ./src/ca_biositing/webservice @@ -8782,6 +9002,24 @@ packages: - trio>=0.32.0 ; python_full_version >= '3.10' and extra == 'trio' - trio>=0.31.0 ; python_full_version < '3.10' and extra == 'trio' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + sha256: 830fc81970cd9d19869909b9b16d241f4d557e4f201a1030aa6ed87c6aa8b930 + md5: 9958d4a1ee7e9c768fe8f4fb51bd07ea + depends: + - exceptiongroup >=1.0.2 + - idna >=2.8 + - python >=3.10 + - typing_extensions >=4.5 + - python + constrains: + - trio >=0.32.0 + - uvloop >=0.21 + license: MIT + license_family: MIT + purls: + - pkg:pypi/anyio?source=hash-mapping + size: 144702 + timestamp: 1764375386926 - conda: https://conda.anaconda.org/conda-forge/osx-64/aom-3.9.1-hf036a51_0.conda sha256: 3032f2f55d6eceb10d53217c2a7f43e1eac83603d91e21ce502e8179e63a75f5 md5: 3f17bc32cb7fcb2b4bf3d8d37f656eb8 @@ -8873,6 +9111,96 @@ packages: - pywin32 ; extra == 'windows' - tzdata ; extra == 'windows' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + sha256: bea62005badcb98b1ae1796ec5d70ea0fc9539e7d59708ac4e7d41e2f4bb0bad + md5: 8ac12aff0860280ee0cff7fa2cf63f3b + depends: + - argon2-cffi-bindings + - python >=3.9 + - typing-extensions + constrains: + - argon2_cffi ==999 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi?source=hash-mapping + size: 18715 + timestamp: 1749017288144 +- conda: https://conda.anaconda.org/conda-forge/linux-64/argon2-cffi-bindings-25.1.0-py312h4c3975b_2.conda + sha256: 7988c207b2b766dad5ebabf25a92b8d75cb8faed92f256fd7a4e0875c9ec6d58 + md5: 1567f06d717246abab170736af8bad1b + depends: + - __glibc >=2.17,<3.0.a0 + - cffi >=1.0.1 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 35646 + timestamp: 1762509443854 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/argon2-cffi-bindings-25.1.0-py312hcd1a082_2.conda + sha256: 4b9bc79be725463b1214ad2735657e9349865fa6980f0ccb892b63dd95d95449 + md5: ac6c4634dc5a01e92f604e55656d9158 + depends: + - cffi >=1.0.1 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python >=3.12,<3.13.0a0 *_cpython + - python_abi 3.12.* *_cp312 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 37418 + timestamp: 1762509514735 +- conda: https://conda.anaconda.org/conda-forge/osx-64/argon2-cffi-bindings-25.1.0-py313hf050af9_2.conda + sha256: e2644e87c26512e38c63ace8fc19120a472c0983718a8aa264862c25294d0632 + md5: 1fedb53ffc72b7d1162daa934ad7996b + depends: + - __osx >=10.13 + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 33301 + timestamp: 1762509795647 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/argon2-cffi-bindings-25.1.0-py313h6535dbc_2.conda + sha256: 05ea6fa7109235cfb4fc24526bae1fe82d88bbb5e697ab3945c313f5f041af5b + md5: e23e087109b2096db4cf9a3985bab329 + depends: + - __osx >=11.0 + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 33947 + timestamp: 1762510144907 +- conda: https://conda.anaconda.org/conda-forge/win-64/argon2-cffi-bindings-25.1.0-py313h5ea7bf4_2.conda + sha256: 3f8a1affdfeb2be5289d709e365fc6e386d734773895215cf8cbc5100fa6af9a + md5: eabb4b677b54874d7d6ab775fdaa3d27 + depends: + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 38779 + timestamp: 1762509796090 - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl name: arrow version: 1.4.0 @@ -8894,6 +9222,20 @@ packages: - pytz==2025.2 ; extra == 'test' - simplejson==3.* ; extra == 'test' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda + sha256: 792da8131b1b53ff667bd6fc617ea9087b570305ccb9913deb36b8e12b3b5141 + md5: 85c4f19f377424eafc4ed7911b291642 + depends: + - python >=3.10 + - python-dateutil >=2.7.0 + - python-tzdata + - python + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/arrow?source=hash-mapping + size: 113854 + timestamp: 1760831179410 - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl name: asgi-lifespan version: 2.1.0 @@ -8927,6 +9269,19 @@ packages: - pkg:pypi/asttokens?source=hash-mapping size: 28797 timestamp: 1763410017955 +- conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda + sha256: 3b7233041e462d9eeb93ea1dfe7b18aca9c358832517072054bb8761df0c324b + md5: d9d0f99095a9bb7e3641bca8c6ad2ac7 + depends: + - python >=3.9 + - typing_extensions >=4.0.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/async-lru?source=hash-mapping + size: 17335 + timestamp: 1742153708859 - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda sha256: 33d12250c870e06c9a313c6663cfbf1c50380b73dfbbb6006688c3134b29b45a md5: 5d842988b11a8c3ab57fb70840c83d24 @@ -10758,6 +11113,43 @@ packages: - xarray ; python_full_version < '3.15' and extra == 'test-tox' - coverage>=5.5 ; extra == 'test-tox-coverage' requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + sha256: bf1e71c3c0a5b024e44ff928225a0874fc3c3356ec1a0b6fe719108e6d1288f6 + md5: 5267bef8efea4127aacd1f4e1f149b6e + depends: + - python >=3.10 + - soupsieve >=1.2 + - typing-extensions + license: MIT + license_family: MIT + purls: + - pkg:pypi/beautifulsoup4?source=hash-mapping + size: 90399 + timestamp: 1764520638652 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + sha256: e03ba1a2b93fe0383c57920a9dc6b4e0c2c7972a3f214d531ed3c21dc8f8c717 + md5: b1a27250d70881943cca0dd6b4ba0956 + depends: + - python >=3.10 + - webencodings + - python + constrains: + - tinycss >=1.1.0,<1.5 + license: Apache-2.0 AND MIT + purls: + - pkg:pypi/bleach?source=hash-mapping + size: 141952 + timestamp: 1763589981635 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda + sha256: f85f6b2c7938d8c20c80ce5b7e6349fafbb49294641b5648273c5f892b150768 + md5: 08a03378bc5293c6f97637323802f480 + depends: + - bleach ==6.3.0 pyhcf101f3_0 + - tinycss2 + license: Apache-2.0 AND MIT + purls: [] + size: 4386 + timestamp: 1763589981639 - conda: https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda sha256: e7af5d1183b06a206192ff440e08db1c4e8b2ca1f8376ee45fb2f3a85d4ee45d md5: 2c2fae981fd2afd00812c92ac47d023d @@ -11317,6 +11709,28 @@ packages: purls: [] size: 152827 timestamp: 1762967310929 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + noarch: python + sha256: 561e6660f26c35d137ee150187d89767c988413c978e1b712d53f27ddf70ea17 + md5: 9b347a7ec10940d3f7941ff6c460b551 + depends: + - cached_property >=1.5.2,<1.5.3.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4134 + timestamp: 1615209571450 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 + sha256: 6dbf7a5070cc43d90a1e4c2ec0c541c69d8e30a0e25f50ce9f6e4a432e42c5d7 + md5: 576d629e47797577ab0f1b351297ef4a + depends: + - python >=3.6 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/cached-property?source=hash-mapping + size: 11065 + timestamp: 1615209567874 - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl name: cachetools version: 6.2.1 @@ -12742,6 +13156,17 @@ packages: - pkg:pypi/decorator?source=hash-mapping size: 14129 timestamp: 1740385067843 +- conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 + sha256: 9717a059677553562a8f38ff07f3b9f61727bd614f505658b0a5ecbcf8df89be + md5: 961b3a227b437d82ad7054484cfa71b2 + depends: + - python >=3.6 + license: PSF-2.0 + license_family: PSF + purls: + - pkg:pypi/defusedxml?source=hash-mapping + size: 24062 + timestamp: 1615232388757 - pypi: https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl name: deprecated version: 1.3.1 @@ -13489,6 +13914,18 @@ packages: requires_dist: - cached-property>=1.3.0 ; python_full_version < '3.8' requires_python: '>=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4,<4' +- conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + sha256: 2509992ec2fd38ab27c7cdb42cf6cadc566a1cc0d1021a2673475d9fa87c6276 + md5: d3549fd50d450b6d9e7dddff25dd2110 + depends: + - cached-property >=1.3.0 + - python >=3.9,<4 + license: MPL-2.0 + license_family: MOZILLA + purls: + - pkg:pypi/fqdn?source=hash-mapping + size: 16705 + timestamp: 1733327494780 - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda sha256: bf8e4dffe46f7d25dc06f31038cacb01672c47b9f45201f065b0f4d00ab0a83e md5: 4afc585cd97ba8a23809406cd8a9eda8 @@ -14972,6 +15409,18 @@ packages: version: 0.16.0 sha256: 63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda + sha256: f64b68148c478c3bfc8f8d519541de7d2616bf59d44485a5271041d40c061887 + md5: 4b69232755285701bc86a5afe4d9933a + depends: + - python >=3.9 + - typing_extensions + license: MIT + license_family: MIT + purls: + - pkg:pypi/h11?source=hash-mapping + size: 37697 + timestamp: 1745526482242 - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda sha256: 84c64443368f84b600bfecc529a1194a3b14c3656ee2e832d15a20e0329b6da3 md5: 164fc43f0b53b6e3a7bc7dce5e4f1dc9 @@ -15222,6 +15671,23 @@ packages: - socksio==1.* ; extra == 'socks' - trio>=0.22.0,<1.0 ; extra == 'trio' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + sha256: 04d49cb3c42714ce533a8553986e1642d0549a05dc5cc48e0d43ff5be6679a5b + md5: 4f14640d58e2cc0aa0819d9d8ba125bb + depends: + - python >=3.9 + - h11 >=0.16 + - h2 >=3,<5 + - sniffio 1.* + - anyio >=4.0,<5.0 + - certifi + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/httpcore?source=hash-mapping + size: 49483 + timestamp: 1745602916758 - conda: https://conda.anaconda.org/conda-forge/noarch/httplib2-0.22.0-pyhd8ed1ab_1.conda sha256: f6ee2956004d98a6ea9ebdaa975dd222b658a31de3ed89348eb67749a0949c6f md5: 2a534e4b838b6efab1732c869d993b45 @@ -15264,6 +15730,21 @@ packages: - socksio==1.* ; extra == 'socks' - zstandard>=0.18.0 ; extra == 'zstd' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda + sha256: cd0f1de3697b252df95f98383e9edb1d00386bfdd03fdf607fa42fe5fcb09950 + md5: d6989ead454181f4f9bc987d3dc4e285 + depends: + - anyio + - certifi + - httpcore 1.* + - idna + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/httpx?source=hash-mapping + size: 63082 + timestamp: 1733663449209 - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl name: humanize version: 4.14.0 @@ -15442,6 +15923,20 @@ packages: - pkg:pypi/importlib-metadata?source=hash-mapping size: 34641 timestamp: 1747934053147 +- conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda + sha256: acc1d991837c0afb67c75b77fdc72b4bf022aac71fedd8b9ea45918ac9b08a80 + md5: c85c76dc67d75619a92f51dfbce06992 + depends: + - python >=3.9 + - zipp >=3.1.0 + constrains: + - importlib-resources >=6.5.2,<6.5.3.0a0 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/importlib-resources?source=hash-mapping + size: 33781 + timestamp: 1736252433366 - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda sha256: e1a9e3b1c8fe62dc3932a616c284b5d8cbe3124bbfbedcf4ce5c828cb166ee19 md5: 9614359868482abba1bd15ce465e3c42 @@ -15608,6 +16103,22 @@ packages: - pkg:pypi/ipython-pygments-lexers?source=hash-mapping size: 13993 timestamp: 1737123723464 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + sha256: 6bb58afb7eabc8b4ac0c7e92707fb498313cc0164cf04e7ba1090dbf49af514b + md5: d68e3f70d1f068f1b66d94822fdc644e + depends: + - comm >=0.1.3 + - ipython >=6.1.0 + - jupyterlab_widgets >=3.0.15,<3.1.0 + - python >=3.10 + - traitlets >=4.3.1 + - widgetsnbextension >=4.0.14,<4.1.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipywidgets?source=hash-mapping + size: 114376 + timestamp: 1762040524661 - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl name: isodate version: 0.7.2 @@ -15620,6 +16131,18 @@ packages: requires_dist: - arrow>=0.15.0 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda + sha256: 08e838d29c134a7684bca0468401d26840f41c92267c4126d7b43a6b533b0aed + md5: 0b0154421989637d424ccf0f104be51a + depends: + - arrow >=0.15.0 + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/isoduration?source=hash-mapping + size: 19832 + timestamp: 1733493720346 - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda sha256: 92c4d217e2dc68983f724aa983cca5464dcb929c566627b26a2511159667dba8 md5: a4f4c5dc9b80bc50e0d3dc4e6e8f1bd9 @@ -15721,6 +16244,16 @@ packages: - click - pyyaml requires_python: '>=3.7.0' +- conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + sha256: ba03ca5a6db38d9f48bd30172e8c512dea7a686a5c7701c6fcdb7b3023dae2ad + md5: 8d5f66ebf832c4ce28d5c37a0e76605c + depends: + - python >=3.10 + license: Apache-2.0 + purls: + - pkg:pypi/json5?source=compressed-mapping + size: 34017 + timestamp: 1767325114901 - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl name: jsonasobj version: 1.3.1 @@ -15744,6 +16277,18 @@ packages: version: 3.0.0 sha256: 13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + sha256: 1a1328476d14dfa8b84dbacb7f7cd7051c175498406dc513ca6c679dc44f3981 + md5: cd2214824e36b0180141d422aba01938 + depends: + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jsonpointer?source=hash-mapping + size: 13967 + timestamp: 1765026384757 - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl name: jsonschema version: 4.25.1 @@ -15771,6 +16316,22 @@ packages: - uri-template ; extra == 'format-nongpl' - webcolors>=24.6.0 ; extra == 'format-nongpl' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + sha256: ac377ef7762e49cb9c4f985f1281eeff471e9adc3402526eea78e6ac6589cf1d + md5: 341fd940c242cf33e832c0402face56f + depends: + - attrs >=22.2.0 + - jsonschema-specifications >=2023.3.6 + - python >=3.9 + - referencing >=0.28.4 + - rpds-py >=0.7.1 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/jsonschema?source=hash-mapping + size: 81688 + timestamp: 1755595646123 - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl name: jsonschema-specifications version: 2025.9.1 @@ -15778,6 +16339,69 @@ packages: requires_dist: - referencing>=0.31.0 requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + sha256: 0a4f3b132f0faca10c89fdf3b60e15abb62ded6fa80aebfc007d05965192aa04 + md5: 439cd0f567d697b20a8f45cb70a1005a + depends: + - python >=3.10 + - referencing >=0.31.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/jsonschema-specifications?source=hash-mapping + size: 19236 + timestamp: 1757335715225 +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + sha256: aef6705fe1335e6472e1b6365fcdb586356b18dceff72d8d6a315fc90e900ccf + md5: 13e31c573c884962318a738405ca3487 + depends: + - jsonschema >=4.25.1,<4.25.2.0a0 + - fqdn + - idna + - isoduration + - jsonpointer >1.13 + - rfc3339-validator + - rfc3986-validator >0.1.0 + - rfc3987-syntax >=1.1.0 + - uri-template + - webcolors >=24.6.0 + license: MIT + license_family: MIT + purls: [] + size: 4744 + timestamp: 1755595646123 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + sha256: b538e15067d05768d1c0532a6d9b0625922a1cce751dd6a2af04f7233a1a70e9 + md5: 9453512288d20847de4356327d0e1282 + depends: + - ipykernel + - ipywidgets + - jupyter_console + - jupyterlab + - nbconvert-core + - notebook + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter?source=hash-mapping + size: 8891 + timestamp: 1733818677113 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda + sha256: 897ad2e2c2335ef3c2826d7805e16002a1fd0d509b4ae0bc66617f0e0ff07bc2 + md5: 62b7c96c6cd77f8173cc5cada6a9acaa + depends: + - importlib-metadata >=4.8.3 + - jupyter_server >=1.1.2 + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-lsp?source=hash-mapping + size: 60377 + timestamp: 1756388269267 - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda sha256: 19d8bd5bb2fde910ec59e081eeb59529491995ce0d653a5209366611023a0b3a md5: 4ebae00eae9705b0c3d6d1018a81d047 @@ -15812,6 +16436,25 @@ packages: - pkg:pypi/jupyter-client?source=compressed-mapping size: 111367 timestamp: 1765375773813 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda + sha256: aee0cdd0cb2b9321d28450aec4e0fd43566efcd79e862d70ce49a68bf0539bcd + md5: 801dbf535ec26508fac6d4b24adfb76e + depends: + - ipykernel >=6.14 + - ipython + - jupyter_client >=7.0.0 + - jupyter_core >=4.12,!=5.0.* + - prompt_toolkit >=3.0.30 + - pygments + - python >=3.9 + - pyzmq >=17 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-console?source=hash-mapping + size: 26874 + timestamp: 1733818130068 - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyh6dadd2b_0.conda sha256: ed709a6c25b731e01563521ef338b93986cd14b5bc17f35e9382000864872ccc md5: a8db462b01221e9f5135be466faeb3e0 @@ -15848,6 +16491,140 @@ packages: - pkg:pypi/jupyter-core?source=hash-mapping size: 65503 timestamp: 1760643864586 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + sha256: 37e6ac3ccf7afcc730c3b93cb91a13b9ae827fd306f35dd28f958a74a14878b5 + md5: f56000b36f09ab7533877e695e4e8cb0 + depends: + - jsonschema-with-format-nongpl >=4.18.0 + - packaging + - python >=3.9 + - python-json-logger >=2.0.4 + - pyyaml >=5.3 + - referencing + - rfc3339-validator + - rfc3986-validator >=0.1.1 + - traitlets >=5.3 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-events?source=hash-mapping + size: 23647 + timestamp: 1738765986736 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + sha256: 74c4e642be97c538dae1895f7052599dfd740d8bd251f727bce6453ce8d6cd9a + md5: d79a87dcfa726bcea8e61275feed6f83 + depends: + - anyio >=3.1.0 + - argon2-cffi >=21.1 + - jinja2 >=3.0.3 + - jupyter_client >=7.4.4 + - jupyter_core >=4.12,!=5.0.* + - jupyter_events >=0.11.0 + - jupyter_server_terminals >=0.4.4 + - nbconvert-core >=6.4.4 + - nbformat >=5.3.0 + - overrides >=5.0 + - packaging >=22.0 + - prometheus_client >=0.9 + - python >=3.10 + - pyzmq >=24 + - send2trash >=1.8.2 + - terminado >=0.8.3 + - tornado >=6.2.0 + - traitlets >=5.6.0 + - websocket-client >=1.7 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server?source=hash-mapping + size: 347094 + timestamp: 1755870522134 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + sha256: 0890fc79422191bc29edf17d7b42cff44ba254aa225d31eb30819f8772b775b8 + md5: 2d983ff1b82a1ccb6f2e9d8784bdd6bd + depends: + - python >=3.9 + - terminado >=0.8.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server-terminals?source=hash-mapping + size: 19711 + timestamp: 1733428049134 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + sha256: ac31a517238173eb565ba9f517b1f9437fba48035f1276a9c1190c145657cafd + md5: f8e8f8db45e1a946ce9b20b0f60b3111 + depends: + - async-lru >=1.0.0 + - httpx >=0.25.0,<1 + - ipykernel >=6.5.0,!=6.30.0 + - jinja2 >=3.0.3 + - jupyter-lsp >=2.0.0 + - jupyter_core + - jupyter_server >=2.4.0,<3 + - jupyterlab_server >=2.28.0,<3 + - notebook-shim >=0.2 + - packaging + - python >=3.10 + - setuptools >=41.1.0 + - tomli >=1.2.2 + - tornado >=6.2.0 + - traitlets + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab?source=hash-mapping + size: 8141875 + timestamp: 1765819955819 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + sha256: dc24b900742fdaf1e077d9a3458fd865711de80bca95fe3c6d46610c532c6ef0 + md5: fd312693df06da3578383232528c468d + depends: + - pygments >=2.4.1,<3 + - python >=3.9 + constrains: + - jupyterlab >=4.0.8,<5.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-pygments?source=hash-mapping + size: 18711 + timestamp: 1733328194037 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + sha256: 381d2d6a259a3be5f38a69463e0f6c5dcf1844ae113058007b51c3bef13a7cee + md5: a63877cb23de826b1620d3adfccc4014 + depends: + - babel >=2.10 + - jinja2 >=3.0.3 + - json5 >=0.9.0 + - jsonschema >=4.18 + - jupyter_server >=1.21,<3 + - packaging >=21.3 + - python >=3.10 + - requests >=2.31 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-server?source=hash-mapping + size: 51621 + timestamp: 1761145478692 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda + sha256: 5c03de243d7ae6247f39a402f4785d95e61c3be79ef18738e8f17155585d31a8 + md5: dbf8b81974504fa51d34e436ca7ef389 + depends: + - python >=3.10 + - python + constrains: + - jupyterlab >=3,<5 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-widgets?source=hash-mapping + size: 216779 + timestamp: 1762267481404 - conda: https://conda.anaconda.org/conda-forge/linux-64/kealib-1.6.2-h74a6b89_0.conda sha256: 7a5f7b5f0be22b9221a0c2dbdb00b5131aadd5a0cbf73318a4e718c36fb93445 md5: fa0c6d118bbd2fe95ffcc419dc075f65 @@ -16055,6 +16832,17 @@ packages: purls: [] size: 604863 timestamp: 1664997611416 +- conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda + sha256: 49570840fb15f5df5d4b4464db8ee43a6d643031a2bc70ef52120a52e3809699 + md5: 9b965c999135d43a3d0f7bd7d024e26a + depends: + - python >=3.10 + license: MIT + license_family: MIT + purls: + - pkg:pypi/lark?source=compressed-mapping + size: 94312 + timestamp: 1761596921009 - conda: https://conda.anaconda.org/conda-forge/linux-64/laz-perf-3.4.0-h00ab1b0_0.conda sha256: 4c130699ffcfddff7073ac079558d3e970b3b6198e096d586183e05d1ee714ac md5: 3c32b1aac6372685e9eb832b8d789b80 @@ -25730,6 +26518,21 @@ packages: - pkg:pypi/markupsafe?source=hash-mapping size: 24541 timestamp: 1759055509267 +- conda: https://conda.anaconda.org/conda-forge/osx-64/markupsafe-3.0.3-py313h0f4d31d_0.conda + sha256: 9c698da56e3bdae80be2a7bc0d19565971b36060155374d16fce14271c8b695c + md5: 884a82dc80ecd251e38d647808c424b3 + depends: + - __osx >=10.13 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + constrains: + - jinja2 >=3.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/markupsafe?source=hash-mapping + size: 25105 + timestamp: 1759055575973 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py312h5748b74_0.conda sha256: b6aadcee6a0b814a0cb721e90575cbbe911b17ec46542460a9416ed2ec1a568e md5: 82221456841d3014a175199e4792465b @@ -25746,6 +26549,22 @@ packages: - pkg:pypi/markupsafe?source=hash-mapping size: 25121 timestamp: 1759055677633 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py313h7d74516_0.conda + sha256: e06902a1bf370fdd4ada0a8c81c504868fdb7e9971b72c6bd395aa4e5a497bd2 + md5: 3df5979cc0b761dda0053ffdb0bca3ea + depends: + - __osx >=11.0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + constrains: + - jinja2 >=3.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/markupsafe?source=hash-mapping + size: 25778 + timestamp: 1759055530601 - conda: https://conda.anaconda.org/conda-forge/win-64/markupsafe-3.0.3-py313hd650c13_0.conda sha256: 988d14095c1392e055fd75e24544da2db01ade73b0c2f99ddc8e2b8678ead4cc md5: 47eaaa4405741beb171ea6edc6eaf874 @@ -25931,6 +26750,19 @@ packages: purls: [] size: 86618 timestamp: 1746450788037 +- conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + sha256: d3fb4beb5e0a52b6cc33852c558e077e1bfe44df1159eb98332d69a264b14bae + md5: b11e360fc4de2b0035fc8aaa74f17fd6 + depends: + - python >=3.10 + - typing_extensions + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/mistune?source=compressed-mapping + size: 74250 + timestamp: 1766504456031 - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.6.1-pyhd8ed1ab_1.conda sha256: 902d2e251f9a7ffa7d86a3e62be5b2395e28614bd4dbe5f50acf921fd64a8c35 md5: 14661160be39d78f2b210f2cc2766059 @@ -26357,6 +27189,66 @@ packages: - fastnumbers>=2.0.0 ; extra == 'fast' - pyicu>=1.0.0 ; extra == 'icu' requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + sha256: 1b66960ee06874ddceeebe375d5f17fb5f393d025a09e15b830ad0c4fffb585b + md5: 00f5b8dafa842e0c27c1cd7296aa4875 + depends: + - jupyter_client >=6.1.12 + - jupyter_core >=4.12,!=5.0.* + - nbformat >=5.1 + - python >=3.8 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbclient?source=compressed-mapping + size: 28473 + timestamp: 1766485646962 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + sha256: 8f575e5c042b17f4677179a6ba474bdbe76573936d3d3e2aeb42b511b9cb1f3f + md5: cfc86ccc3b1de35d36ccaae4c50391f5 + depends: + - beautifulsoup4 + - bleach-with-css !=5.0.0 + - defusedxml + - importlib-metadata >=3.6 + - jinja2 >=3.0 + - jupyter_core >=4.7 + - jupyterlab_pygments + - markupsafe >=2.0 + - mistune >=2.0.3,<4 + - nbclient >=0.5.0 + - nbformat >=5.7 + - packaging + - pandocfilters >=1.4.1 + - pygments >=2.4.1 + - python >=3.10 + - traitlets >=5.1 + - python + constrains: + - pandoc >=2.9.2,<4.0.0 + - nbconvert ==7.16.6 *_1 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbconvert?source=compressed-mapping + size: 199273 + timestamp: 1760797634443 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda + sha256: 7a5bd30a2e7ddd7b85031a5e2e14f290898098dc85bea5b3a5bf147c25122838 + md5: bbe1963f1e47f594070ffe87cdf612ea + depends: + - jsonschema >=2.6 + - jupyter_core >=4.12,!=5.0.* + - python >=3.9 + - python-fastjsonschema >=2.15 + - traitlets >=5.1 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbformat?source=hash-mapping + size: 100945 + timestamp: 1733402844974 - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 md5: 47e340acb35de30501a76c7c799c41d7 @@ -26569,6 +27461,36 @@ packages: - pkg:pypi/nose2?source=hash-mapping size: 92820 timestamp: 1580623268425 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + sha256: 672ec7db73c8bfbacf9227c0c2287effdeded77b4d06373f2e498a310ce76a8c + md5: c984a8b773a34e38f5cf399b6d582e5c + depends: + - importlib_resources >=5.0 + - jupyter_server >=2.4.0,<3 + - jupyterlab >=4.5.1,<4.6 + - jupyterlab_server >=2.28.0,<3 + - notebook-shim >=0.2,<0.3 + - python >=3.10 + - tornado >=6.2.0 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook?source=hash-mapping + size: 10040377 + timestamp: 1765875192987 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda + sha256: 7b920e46b9f7a2d2aa6434222e5c8d739021dbc5cc75f32d124a8191d86f9056 + md5: e7f89ea5f7ea9401642758ff50a2d9c1 + depends: + - jupyter_server >=1.8,<3 + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook-shim?source=hash-mapping + size: 16817 + timestamp: 1733408419340 - conda: https://conda.anaconda.org/conda-forge/linux-64/nspr-4.37-h29cc59b_0.conda sha256: 472306630dcd49a221863b91bd89f5b8b81daf1b99adf1968c9f12021176d873 md5: d73ccc379297a67ed921bd55b38a6c6a @@ -27417,6 +28339,18 @@ packages: version: 3.11.5 sha256: 2b91126e7b470ff2e75746f6f6ee32b9ab67b7a93c8ba1d15d3a0caaf16ec875 requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda + sha256: 1840bd90d25d4930d60f57b4f38d4e0ae3f5b8db2819638709c36098c6ba770c + md5: e51f1e4089cad105b6cac64bd8166587 + depends: + - python >=3.9 + - typing_utils + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/overrides?source=hash-mapping + size: 30139 + timestamp: 1734587755455 - conda: https://conda.anaconda.org/conda-forge/noarch/owslib-0.34.1-pyhd8ed1ab_0.conda sha256: bd1dc309ec0d161cd1f65279cfc548c2b9dda7dc8cf44277b303923cfdf40386 md5: 819c0171e7092a83806a7ef9158f9158 @@ -28654,6 +29588,17 @@ packages: - pandas>=0.23 - xarray requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 + sha256: 2bb9ba9857f4774b85900c2562f7e711d08dd48e2add9bee4e1612fbee27e16f + md5: 457c2c8c08e54905d6954e79cb5b5db9 + depends: + - python !=3.0,!=3.1,!=3.2,!=3.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pandocfilters?source=hash-mapping + size: 11627 + timestamp: 1631603397334 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda sha256: ce76d5a1fc6c7ef636cbdbf14ce2d601a1bfa0dd8d286507c1fd02546fccb94b md5: 1a884d2b1ea21abfb73911dcdb8342e4 @@ -29610,6 +30555,17 @@ packages: requires_dist: - twisted ; extra == 'twisted' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda + sha256: 13dc67de68db151ff909f2c1d2486fa7e2d51355b25cee08d26ede1b62d48d40 + md5: a1e91db2d17fd258c64921cb38e6745a + depends: + - python >=3.10 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/prometheus-client?source=hash-mapping + size: 54592 + timestamp: 1758278323953 - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda sha256: 4817651a276016f3838957bfdf963386438c70761e9faec7749d411635979bae md5: edb16f14d920fb3faf17f5ce582942d6 @@ -29624,6 +30580,16 @@ packages: - pkg:pypi/prompt-toolkit?source=hash-mapping size: 273927 timestamp: 1756321848365 +- conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda + sha256: e79922a360d7e620df978417dd033e66226e809961c3e659a193f978a75a9b0b + md5: 6d034d3a6093adbba7b24cb69c8c621e + depends: + - prompt-toolkit >=3.0.52,<3.0.53.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 7212 + timestamp: 1756321849562 - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.1.3-py312h5253ce2_0.conda sha256: 1b679202ebccf47be64509a4fc2a438a66229403257630621651b2886b882597 md5: 82ce56c5a4a55165aed95e04923ab363 @@ -30506,6 +31472,68 @@ packages: - pkg:pypi/pynacl?source=hash-mapping size: 1184528 timestamp: 1764063741665 +- conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-core-12.1-py313h07bcf3a_0.conda + sha256: 1e0edd34b804e20ba064dcebcfce3066d841ec812f29ed65902da7192af617d1 + md5: 6a2c3a617a70f97ca53b7b88461b1c27 + depends: + - __osx >=10.13 + - libffi >=3.5.2,<3.6.0a0 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - setuptools + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-core?source=hash-mapping + size: 491157 + timestamp: 1763151415674 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-core-12.1-py313h40b429f_0.conda + sha256: 307ca29ebf2317bd2561639b1ee0290fd8c03c3450fa302b9f9437d8df6a5280 + md5: 31a0a72f3466682d0ea2ebcbd7d319b8 + depends: + - __osx >=11.0 + - libffi >=3.5.2,<3.6.0a0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + - setuptools + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-core?source=hash-mapping + size: 481508 + timestamp: 1763152124940 +- conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-framework-cocoa-12.1-py313hf669bc3_0.conda + sha256: 4761b8448bb9ecfcd9636a506104e6474e0f4cb73d108f2088997702ae984a00 + md5: 628b5ad83d6140fe4bfa937e2f357ed7 + depends: + - __osx >=10.13 + - libffi >=3.5.2,<3.6.0a0 + - pyobjc-core 12.1.* + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-framework-cocoa?source=hash-mapping + size: 374120 + timestamp: 1763160397755 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-framework-cocoa-12.1-py313hcc5defa_0.conda + sha256: 194e188d8119befc952d04157079733e2041a7a502d50340ddde632658799fdc + md5: a6d28c8fc266a3d3c3dae183e25c4d31 + depends: + - __osx >=11.0 + - libffi >=3.5.2,<3.6.0a0 + - pyobjc-core 12.1.* + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-framework-cocoa?source=hash-mapping + size: 376136 + timestamp: 1763160678792 - pypi: https://files.pythonhosted.org/packages/c8/23/c8dd17cbb1bd6614f306a983e260e31c01f3e8e8cc8954ba68749db6ae82/pyparsing-3.3.0-py3-none-any.whl name: pyparsing version: 3.3.0 @@ -31352,6 +32380,18 @@ packages: - pkg:pypi/python-dotenv?source=hash-mapping size: 26922 timestamp: 1761503229008 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda + sha256: df9aa74e9e28e8d1309274648aac08ec447a92512c33f61a8de0afa9ce32ebe8 + md5: 23029aae904a2ba587daba708208012f + depends: + - python >=3.9 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/fastjsonschema?source=hash-mapping + size: 244628 + timestamp: 1755304154927 - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda sha256: 59f17182813f8b23709b7d4cfda82c33b72dd007cb729efa0033c609fbd92122 md5: c20172b4c59fbe288fa50cdc1b693d73 @@ -31408,6 +32448,17 @@ packages: - mkdocs-literate-nav ; extra == 'dev' - mike ; extra == 'dev' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + sha256: 4790787fe1f4e8da616edca4acf6a4f8ed4e7c6967aa31b920208fc8f95efcca + md5: a61bf9ec79426938ff785eb69dbb1960 + depends: + - python >=3.6 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/python-json-logger?source=hash-mapping + size: 13383 + timestamp: 1677079727691 - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl name: python-slugify version: 8.0.4 @@ -31544,6 +32595,22 @@ packages: purls: [] size: 4856 timestamp: 1646866525560 +- conda: https://conda.anaconda.org/conda-forge/win-64/pywinpty-2.0.15-py313h5813708_1.conda + sha256: d34a7cd0a4a7dc79662cb6005e01d630245d9a942e359eb4d94b2fb464ed2552 + md5: 8f01ed27e2baa455e753301218e054fd + depends: + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + - winpty + license: MIT + license_family: MIT + purls: + - pkg:pypi/pywinpty?source=hash-mapping + size: 216075 + timestamp: 1759556799508 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py312h8a5da7c_0.conda sha256: 1b3dc4c25c83093fff08b86a3574bc6b94ba355c8eba1f35d805c5e256455fc7 md5: fba10c2007c8b06f77c5a23ce3a635ad @@ -33180,6 +34247,21 @@ packages: - rpds-py>=0.7.0 - typing-extensions>=4.4.0 ; python_full_version < '3.13' requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda + sha256: 0577eedfb347ff94d0f2fa6c052c502989b028216996b45c7f21236f25864414 + md5: 870293df500ca7e18bedefa5838a22ab + depends: + - attrs >=22.2.0 + - python >=3.10 + - rpds-py >=0.7.0 + - typing_extensions >=4.4.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/referencing?source=hash-mapping + size: 51788 + timestamp: 1760379115194 - pypi: https://files.pythonhosted.org/packages/4c/93/181070cd1aa2fa541ff2d3afcf763ceecd4937b34c615fa92765020a6c90/regex-2025.10.23-cp313-cp313-macosx_10_13_x86_64.whl name: regex version: 2025.10.23 @@ -33263,10 +34345,46 @@ packages: requires_dist: - six requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + sha256: 2e4372f600490a6e0b3bac60717278448e323cab1c0fecd5f43f7c56535a99c5 + md5: 36de09a8d3e5d5e6f4ee63af49e59706 + depends: + - python >=3.9 + - six + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3339-validator?source=hash-mapping + size: 10209 + timestamp: 1733600040800 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + sha256: 2a5b495a1de0f60f24d8a74578ebc23b24aa53279b1ad583755f223097c41c37 + md5: 912a71cc01012ee38e6b90ddd561e36f + depends: + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3986-validator?source=hash-mapping + size: 7818 + timestamp: 1598024297745 - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl name: rfc3987 version: 1.3.8 sha256: 10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + sha256: 70001ac24ee62058557783d9c5a7bbcfd97bd4911ef5440e3f7a576f9e43bc92 + md5: 7234f99325263a5af6d4cd195035e8f2 + depends: + - python >=3.9 + - lark >=1.2.2 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3987-syntax?source=hash-mapping + size: 22913 + timestamp: 1752876729969 - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl name: rich version: 14.2.0 @@ -33331,6 +34449,83 @@ packages: version: 0.30.0 sha256: 806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15 requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.30.0-py312h868fb18_0.conda + sha256: 62f46e85caaba30b459da7dfcf3e5488ca24fd11675c33ce4367163ab191a42c + md5: 3ffc5a3572db8751c2f15bacf6a0e937 + depends: + - python + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python_abi 3.12.* *_cp312 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 383750 + timestamp: 1764543174231 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rpds-py-0.30.0-py312h75d7d99_0.conda + sha256: 51e7d33b71b30ac5ceab09cc35521eccdaf4e3897ca4c6eda1059fb82a91b285 + md5: 85212b0e327723285cc36efddd25e03d + depends: + - python + - libgcc >=14 + - python_abi 3.12.* *_cp312 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 380599 + timestamp: 1764543504405 +- conda: https://conda.anaconda.org/conda-forge/osx-64/rpds-py-0.30.0-py313hcc225dc_0.conda + sha256: 8955e67a30f44fbfd390374ba27f445b9e56818b023ccb8fe8f0cd00bec03caa + md5: 7c8790b86262342a2c4f4c9709cf61ae + depends: + - python + - __osx >=10.13 + - python_abi 3.13.* *_cp313 + constrains: + - __osx >=10.13 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 370868 + timestamp: 1764543169321 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/rpds-py-0.30.0-py313h2c089d5_0.conda + sha256: db63344f91e8bfe77703c6764aa9eeafb44d165e286053214722814eabda0264 + md5: 190c2d0d4e98ec97df48cdb74caf44d8 + depends: + - python + - __osx >=11.0 + - python 3.13.* *_cp313 + - python_abi 3.13.* *_cp313 + constrains: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 358961 + timestamp: 1764543165314 +- conda: https://conda.anaconda.org/conda-forge/win-64/rpds-py-0.30.0-py313hfbe8231_0.conda + sha256: 27bd383787c0df7a0a926b11014fd692d60d557398dcf1d50c55aa2378507114 + md5: 58ae648b12cfa6df3923b5fd219931cb + depends: + - python + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 243419 + timestamp: 1764543047271 - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl name: rsa version: 4.9.1 @@ -33876,6 +35071,44 @@ packages: version: 3.0.4 sha256: 9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda + sha256: 5893e203cb099c784bf5b08d29944b5402beebcc361d55e54b676e9b355c7844 + md5: dcff6f8ea9e86a0bda978b88f89f2310 + depends: + - __osx + - pyobjc-framework-cocoa + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=compressed-mapping + size: 22782 + timestamp: 1767192019917 +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh6dadd2b_0.conda + sha256: f154f702baf550de9c1e3517f110bb71a056df5645027c8d15b37f3ea33722cc + md5: 40df72e963d80a403c1861ae9428b13c + depends: + - __win + - pywin32 + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=hash-mapping + size: 22947 + timestamp: 1767192046046 +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda + sha256: 27cd93b4f848a1c8193a7b1b8e6e6d03321462e96997ce95ea1a39305f7ac7cb + md5: f2cc28627a451a28ddd5ef5ab0bf579d + depends: + - __linux + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=hash-mapping + size: 24215 + timestamp: 1767192001650 - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda sha256: 972560fcf9657058e3e1f97186cc94389144b46dbdf58c807ce62e83f977e863 md5: 4de79c071274a53dcaf2a8c749d1499e @@ -34154,6 +35387,17 @@ packages: version: 1.3.1 sha256: 2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + sha256: dce518f45e24cd03f401cb0616917773159a210c19d601c5f2d4e0e5879d30ad + md5: 03fe290994c5e4ec17293cfb6bdce520 + depends: + - python >=3.10 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/sniffio?source=compressed-mapping + size: 15698 + timestamp: 1762941572482 - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl name: snowballstemmer version: 3.0.1 @@ -34176,6 +35420,17 @@ packages: name: sortedcontainers version: 2.4.0 sha256: a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 +- conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda + sha256: 4ba9b8c45862e54d05ed9a04cc6aab5a17756ab9865f57cbf2836e47144153d2 + md5: 7de28c27fe620a4f7dbfaea137c6232b + depends: + - python >=3.10 + license: MIT + license_family: MIT + purls: + - pkg:pypi/soupsieve?source=compressed-mapping + size: 37951 + timestamp: 1766075884412 - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl name: sparqlslurper version: 0.5.1 @@ -34994,6 +36249,36 @@ packages: purls: [] size: 155714 timestamp: 1762510341121 +- conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh6dadd2b_1.conda + sha256: b375e8df0d5710717c31e7c8e93c025c37fa3504aea325c7a55509f64e5d4340 + md5: e43ca10d61e55d0a8ec5d8c62474ec9e + depends: + - __win + - pywinpty >=1.1.0 + - python >=3.10 + - tornado >=6.1.0 + - python + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/terminado?source=hash-mapping + size: 23665 + timestamp: 1766513806974 +- conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda + sha256: 6b6727a13d1ca6a23de5e6686500d0669081a117736a87c8abf444d60c1e40eb + md5: 17b43cee5cc84969529d5d0b0309b2cb + depends: + - __unix + - ptyprocess + - python >=3.10 + - tornado >=6.1.0 + - python + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/terminado?source=hash-mapping + size: 24749 + timestamp: 1766513766867 - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda sha256: df6e7c890567e8f2f4721de6573e685bd917eab17fcf662990bb0ebea6008a6e md5: 5c433e9a89d0f7de90be138e89568bd4 @@ -35190,6 +36475,19 @@ packages: purls: [] size: 2858595 timestamp: 1766191991759 +- conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda + sha256: 7c803480dbfb8b536b9bf6287fa2aa0a4f970f8c09075694174eb4550a4524cd + md5: c0d0b883e97906f7524e2aac94be0e0d + depends: + - python >=3.10 + - webencodings >=0.4 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/tinycss2?source=compressed-mapping + size: 30571 + timestamp: 1764621508086 - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda sha256: a84ff687119e6d8752346d1d408d5cf360dee0badd487a472aa8ddedfdc219e1 md5: a0116df4f4ed05c303811a837d5b39d8 @@ -35406,6 +36704,17 @@ packages: - pkg:pypi/typing-extensions?source=hash-mapping size: 51692 timestamp: 1756220668932 +- conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda + sha256: 3088d5d873411a56bf988eee774559335749aed6f6c28e07bf933256afb9eb6c + md5: f6d7aa696c67756a650e91e15e88223c + depends: + - python >=3.9 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/typing-utils?source=hash-mapping + size: 15183 + timestamp: 1733331395943 - conda: https://conda.anaconda.org/conda-forge/linux-64/tzcode-2025b-hb9d3cd8_0.conda sha256: 324976aab17bee85979761f89457b6a43c11bd93dcebf950d65576b2ab44dd94 md5: 83aa65f939a5cf4a82bfa510cbc38b3f @@ -35808,6 +37117,17 @@ packages: - flake8-use-fstring ; extra == 'dev' - pep8-naming ; extra == 'dev' requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda + sha256: e0eb6c8daf892b3056f08416a96d68b0a358b7c46b99c8a50481b22631a4dfc0 + md5: e7cb0f5745e4c5035a460248334af7eb + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/uri-template?source=hash-mapping + size: 23990 + timestamp: 1733323714454 - conda: https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda sha256: 2aad2aeff7c69a2d7eecd7b662eef756b27d6a6b96f3e2c2a7071340ce14543e md5: d71d3a66528853c0a1ac2c02d79a0284 @@ -36148,6 +37468,28 @@ packages: version: 25.10.0 sha256: 032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + sha256: 21f6c8a20fe050d09bfda3fb0a9c3493936ce7d6e1b3b5f8b01319ee46d6c6f6 + md5: 6639b6b0d8b5a284f027a2003669aa65 + depends: + - python >=3.10 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webcolors?source=hash-mapping + size: 18987 + timestamp: 1761899393153 +- conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda + sha256: 19ff205e138bb056a46f9e3839935a2e60bd1cf01c8241a5e172a422fed4f9c6 + md5: 2841eb5bfc75ce15e9a0054b98dcd64d + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webencodings?source=hash-mapping + size: 15496 + timestamp: 1733236131358 - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda sha256: 42a2b61e393e61cdf75ced1f5f324a64af25f347d16c60b14117393a98656397 md5: 2f1ed718fcd829c184a6d4f0f2e07409 @@ -36260,6 +37602,17 @@ packages: - tzdata>=2020.1 ; sys_platform == 'win32' - tzlocal>=4.0 ; sys_platform != 'darwin' and sys_platform != 'linux' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda + sha256: 826af5e2c09e5e45361fa19168f46ff524e7a766022615678c3a670c45895d9a + md5: dc257b7e7cad9b79c1dfba194e92297b + depends: + - python >=3.10 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/widgetsnbextension?source=hash-mapping + size: 889195 + timestamp: 1762040404362 - conda: https://conda.anaconda.org/conda-forge/noarch/win_inet_pton-1.1.0-pyh7428d3b_8.conda sha256: 93807369ab91f230cf9e6e2a237eaa812492fe00face5b38068735858fba954f md5: 46e441ba871f524e2b067929da3051c2 @@ -36271,6 +37624,13 @@ packages: - pkg:pypi/win-inet-pton?source=hash-mapping size: 9555 timestamp: 1733130678956 +- conda: https://conda.anaconda.org/conda-forge/win-64/winpty-0.4.3-4.tar.bz2 + sha256: 9df10c5b607dd30e05ba08cbd940009305c75db242476f4e845ea06008b0a283 + md5: 1cee351bf20b830d991dbe0bc8cd7dfe + license: MIT + license_family: MIT + purls: [] + size: 1176306 - conda: https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.17.3-py312h4c3975b_1.conda sha256: 8320d5af37eb8933e5d129884ea013b2687e75b08aff5216193df3378eaea92f md5: 8af3faf88325836e46c6cb79828e058c diff --git a/pixi.toml b/pixi.toml index 9f1deb56..345ae21c 100644 --- a/pixi.toml +++ b/pixi.toml @@ -242,6 +242,7 @@ frontend-build = { cmd = "npm run build", cwd = "frontend", description = "Build # Kernel Dependencies # This is to enable Jupyter Notebooks with IPython kernel [feature.kernel.dependencies] +jupyter = ">=1.0.0,<2" ipykernel = ">=7.1.0,<8" ipython = ">=9.0.0,<10" diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb new file mode 100644 index 00000000..18fb0314 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -0,0 +1,1787 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ETL Notebook for CA Biositing Project\n", + "\n", + "This notebook provides a documented walkthrough of the ETL (Extract, Transform, Load) process for the CA Biositing project. It is designed for interactive development and exploration before migrating logic into the production pipeline.\n", + "\n", + "It covers:\n", + "\n", + "1. **Setup**: Importing necessary libraries and establishing a connection to the database.\n", + "2. **Extraction**: Pulling raw data from Google Sheets.\n", + "3. **Cleaning**: Standardizing data types, handling missing values, and cleaning column names.\n", + "4. **Normalization**: Replacing human-readable names (e.g., \"Corn\") with database foreign key IDs (e.g., `resource_id: 1`).\n", + "5. **Utilities**: Common functions for data manipulation and analysis.\n", + "6. **Deployment Plan**: A step-by-step guide for moving the code from this notebook into the production ETL modules." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-04 08:53:57,422 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", + "2026-01-04 08:53:57,423 - INFO - Successfully imported all project modules.\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "import logging\n", + "from IPython.display import display\n", + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "\n", + "# --- Basic Logging Configuration for Notebook ---\n", + "# When running in a notebook, we use Python's standard logging.\n", + "# In the production pipeline, this will be replaced by Prefect's `get_run_logger()`\n", + "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n", + "logger = logging.getLogger()\n", + "\n", + "# --- Robustly find the project root ---\n", + "# This ensures that the notebook can be run from any directory within the project.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# Add the project root to the Python path to allow for module imports\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " logger.info(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " logger.info(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import project modules ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + " from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import replace_name_with_id_df\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " logger.info('Successfully imported all project modules.')\n", + "except ImportError as e:\n", + " logger.error(f'Failed to import project modules: {e}', exc_info=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Cleaning Function" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " \"\"\"Cleans and standardizes a DataFrame extracted from Google Sheets.\n", + "\n", + " This function performs several key operations:\n", + " 1. Cleans column names to a standard format (snake_case).\n", + " 2. Drops rows where essential columns ('repl_no', 'value') are empty.\n", + " 3. Coerces data types for numeric and datetime columns, handling errors gracefully.\n", + " 4. Converts remaining columns to the best possible data types.\n", + "\n", + " Args:\n", + " df (pd.DataFrame): The raw DataFrame.\n", + "\n", + " Returns:\n", + " pd.DataFrame: The cleaned DataFrame.\n", + " \"\"\"\n", + " logger.info('Starting DataFrame cleaning process.')\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.error('Input is not a pandas DataFrame.')\n", + " return None\n", + " \n", + " try:\n", + " # 1. Clean names and drop rows with missing essential data\n", + " df_cleaned = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + " logger.info(f'Dropped {len(df) - len(df_cleaned)} rows with missing values.')\n", + "\n", + " # 2. Coerce numeric types\n", + " df_cleaned['repl_no'] = pd.to_numeric(df_cleaned['repl_no'], errors='coerce').astype('Int32')\n", + " df_cleaned['value'] = pd.to_numeric(df_cleaned['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Coerce datetime types\n", + " if 'created_at' in df_cleaned.columns:\n", + " df_cleaned['created_at'] = pd.to_datetime(df_cleaned['created_at'], errors='coerce')\n", + " if 'updated_at' in df_cleaned.columns:\n", + " df_cleaned['updated_at'] = pd.to_datetime(df_cleaned['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert other dtypes to best possible\n", + " df_cleaned = df_cleaned.convert_dtypes()\n", + " logger.info('Successfully cleaned DataFrame.')\n", + " \n", + " \n", + " # 5. Convert all string data to lowercase\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + " logger.info('Converted all string data to lowercase.')\n", + " return df_cleaned\n", + "\n", + " except Exception as e:\n", + " logger.error(f'An error occurred during DataFrame cleaning: {e}', exc_info=True)\n", + " return None\n", + "\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Normalization Function" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_dataframes(dataframes, normalize_columns):\n", + " \"\"\"Normalizes a list of DataFrames by replacing name columns with foreign key IDs.\n", + "\n", + " This function iterates through a list of dataframes and, for each one, iterates\n", + " through a dictionary of columns that need to be normalized. It uses the \n", + " `replace_name_with_id_df` utility to look up or create the corresponding ID\n", + " in the database.\n", + "\n", + " Args:\n", + " dataframes (list[pd.DataFrame]): A list of DataFrames to normalize.\n", + " normalize_columns (dict): A dictionary mapping column names to SQLModel classes and attributes.\n", + "\n", + " Returns:\n", + " list[pd.DataFrame]: The list of normalized DataFrames.\n", + " \"\"\"\n", + " logger.info(f'Starting normalization process for {len(dataframes)} dataframes.')\n", + " normalized_dfs = []\n", + " try:\n", + " with Session(engine) as db:\n", + " for i, df in enumerate(dataframes):\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.warning(f'Item {i+1} is not a DataFrame, skipping.')\n", + " continue\n", + " \n", + " logger.info(f'Processing DataFrame #{i+1} with {len(df)} rows.')\n", + " df_normalized = df.copy()\n", + "\n", + " for df_col, (model, model_name_attr) in normalize_columns.items():\n", + " if df_col not in df_normalized.columns:\n", + " logger.warning(f\"Column '{df_col}' not in DataFrame #{i+1}. Skipping normalization for this column.\")\n", + " continue\n", + " \n", + " try:\n", + " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name='id',\n", + " final_column_name=f'{df_col}_id'\n", + " )\n", + " new_col_name = f'{df_col}_id'\n", + " num_nulls = df_normalized[new_col_name].isnull().sum()\n", + " logger.info(f\"Successfully normalized '{df_col}'. New column '{new_col_name}' contains {num_nulls} null values.\")\n", + " except Exception as e:\n", + " logger.error(f\"Error normalizing column '{df_col}' in DataFrame #{i+1}: {e}\", exc_info=True)\n", + " continue # Continue to the next column\n", + " \n", + " normalized_dfs.append(df_normalized)\n", + " logger.info(f'Finished processing DataFrame #{i+1}.')\n", + " \n", + " logger.info('Committing database session.')\n", + " db.commit()\n", + " logger.info('Database commit successful.')\n", + " except Exception as e:\n", + " logger.error(f'A critical error occurred during the database session: {e}', exc_info=True)\n", + " db.rollback()\n", + " logger.info('Database session rolled back.')\n", + " \n", + " return normalized_dfs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ETL Execution Example" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-04 08:58:52,849 - INFO - Starting data extraction...\n", + "2026-01-04 08:58:52,865 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:52,877 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:54,699 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:54,703 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:54,724 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:54,736 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:55,992 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:55,996 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:56,016 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:56,029 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:57,291 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:57,294 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:57,296 - INFO - Data extraction complete.\n", + "2026-01-04 08:58:57,299 - INFO - Starting data cleaning...\n", + "2026-01-04 08:58:57,299 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,302 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,309 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,314 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,315 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,316 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,319 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,320 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,321 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,325 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,335 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,340 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,342 - INFO - Data cleaning complete.\n", + "2026-01-04 08:58:57,342 - INFO - Starting data normalization...\n", + "2026-01-04 08:58:57,343 - INFO - Starting normalization process for 3 dataframes.\n", + "2026-01-04 08:58:57,343 - INFO - Processing DataFrame #1 with 759 rows.\n", + "2026-01-04 08:58:57,344 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,355 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,356 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,367 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,367 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,369 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,369 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,372 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,372 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,374 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,374 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,375 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,375 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,379 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,379 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,379 - INFO - Finished processing DataFrame #1.\n", + "2026-01-04 08:58:57,379 - INFO - Processing DataFrame #2 with 64 rows.\n", + "2026-01-04 08:58:57,380 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,384 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,384 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,388 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,388 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,392 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,392 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,396 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,396 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,398 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,398 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,399 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,400 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,403 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,403 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,403 - INFO - Finished processing DataFrame #2.\n", + "2026-01-04 08:58:57,403 - INFO - Processing DataFrame #3 with 390 rows.\n", + "2026-01-04 08:58:57,404 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,406 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,406 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,408 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,408 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,409 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,410 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,411 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,411 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,413 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,413 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,415 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,415 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,416 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,416 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,417 - INFO - Finished processing DataFrame #3.\n", + "2026-01-04 08:58:57,417 - INFO - Committing database session.\n", + "2026-01-04 08:58:57,418 - INFO - Database commit successful.\n", + "2026-01-04 08:58:57,418 - INFO - Data normalization complete.\n", + "2026-01-04 08:58:57,418 - INFO - Displaying results of normalization...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 1 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prox_uuid_031", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "source_codename", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "float64", + "type": "float" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "e2d8eb1c-3fc8-4b97-99b0-df76b598d9df", + "rows": [ + [ + "0", + "d7965110-407f-e356-d41d-b3b9a2b7b7", + "(73)b7b7", + "oakleaf", + "4c", + "prox01xk", + "1.0", + "prox01xk(73)1", + "61.849998474121094", + "2024-10-02 10:31:01", + null, + "pass", + "not ready", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "1", + "c8fea984-2e9a-8def-55fb-1a9d7d9ba8", + "(73)9ba8", + "oakleaf", + "4c", + "prox01xk", + "2.0", + "prox01xk(73)2", + "63.209999084472656", + "2024-10-02 10:31:31", + null, + "pass", + "ready", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "2", + "df304d5d-3a85-4881-7142-6d4e5f957d", + "(73)957d", + "oakleaf", + "4c", + "prox01xk", + "3.0", + "prox01xk(73)3", + "63.27000045776367", + "2024-10-02 10:32:01", + null, + "pass", + "imported", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "3", + "01c6c5be-cea6-54af-3924-b0bad69335", + "(73)9335", + "oakleaf", + "4c", + "prox01xk", + "1.0", + "prox01xk(73)1", + "0.6899999976158142", + "2024-10-03 10:31:01", + null, + "pass", + "import failed", + "", + "24", + "109", + "7", + "23", + "1", + "1", + "6" + ], + [ + "4", + "126745c7-dd41-2f6d-0dc5-28dbca415f", + "(73)415f", + "oakleaf", + "4c", + "prox01xk", + "2.0", + "prox01xk(73)2", + "0.8899999856948853", + "2024-10-03 10:31:31", + null, + "pass", + "", + "", + "24", + "109", + "7", + "23", + "1", + "1", + "6" + ] + ], + "shape": { + "columns": 20, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
prox_uuid_031record_idsource_codenamestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_resultupload_statusnoteresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
0d7965110-407f-e356-d41d-b3b9a2b7b7(73)b7b7oakleaf4cprox01xk1.0prox01xk(73)161.8499982024-10-02 10:31:01NaTpassnot ready24109725116
1c8fea984-2e9a-8def-55fb-1a9d7d9ba8(73)9ba8oakleaf4cprox01xk2.0prox01xk(73)263.2099992024-10-02 10:31:31NaTpassready24109725116
2df304d5d-3a85-4881-7142-6d4e5f957d(73)957doakleaf4cprox01xk3.0prox01xk(73)363.2700002024-10-02 10:32:01NaTpassimported24109725116
301c6c5be-cea6-54af-3924-b0bad69335(73)9335oakleaf4cprox01xk1.0prox01xk(73)10.6900002024-10-03 10:31:01NaTpassimport failed24109723116
4126745c7-dd41-2f6d-0dc5-28dbca415f(73)415foakleaf4cprox01xk2.0prox01xk(73)20.8900002024-10-03 10:31:31NaTpass24109723116
\n", + "
" + ], + "text/plain": [ + " prox_uuid_031 record_id source_codename storage_cond \\\n", + "0 d7965110-407f-e356-d41d-b3b9a2b7b7 (73)b7b7 oakleaf 4c \n", + "1 c8fea984-2e9a-8def-55fb-1a9d7d9ba8 (73)9ba8 oakleaf 4c \n", + "2 df304d5d-3a85-4881-7142-6d4e5f957d (73)957d oakleaf 4c \n", + "3 01c6c5be-cea6-54af-3924-b0bad69335 (73)9335 oakleaf 4c \n", + "4 126745c7-dd41-2f6d-0dc5-28dbca415f (73)415f oakleaf 4c \n", + "\n", + " exper_abbrev repl_no repl_id value created_at \\\n", + "0 prox01xk 1.0 prox01xk(73)1 61.849998 2024-10-02 10:31:01 \n", + "1 prox01xk 2.0 prox01xk(73)2 63.209999 2024-10-02 10:31:31 \n", + "2 prox01xk 3.0 prox01xk(73)3 63.270000 2024-10-02 10:32:01 \n", + "3 prox01xk 1.0 prox01xk(73)1 0.690000 2024-10-03 10:31:01 \n", + "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", + "\n", + " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", + "0 NaT pass not ready 24 109 \n", + "1 NaT pass ready 24 109 \n", + "2 NaT pass imported 24 109 \n", + "3 NaT pass import failed 24 109 \n", + "4 NaT pass 24 109 \n", + "\n", + " preparation_method_id parameter_id unit_id analyst_email_id \\\n", + "0 7 25 1 1 \n", + "1 7 25 1 1 \n", + "2 7 25 1 1 \n", + "3 7 23 1 1 \n", + "4 7 23 1 1 \n", + "\n", + " analysis_type_id \n", + "0 6 \n", + "1 6 \n", + "2 6 \n", + "3 6 \n", + "4 6 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 2 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ult_uuid_037", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "ult_sample_name", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "323960ff-b723-432d-b7c9-fa0f9dd8c185", + "rows": [ + [ + "0", + "421617a5-e50e-9642-2974-b3275fe822", + ")u22e822", + "hum-almhu023km2(15)u22", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(15)1", + "88.5999984741211", + null, + null, + "pass", + "1", + "", + "", + "", + "8", + "80", + "6", + "28", + "3", + "2", + "7" + ], + [ + "1", + "7e7919c2-5db4-6bef-75e2-7e51321200", + ")u001200", + "hum-almhu023km2(15)u00", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(15)1", + null, + null, + null, + "fail", + " 1 dup", + "", + "", + "", + "8", + "80", + "6", + "28", + "3", + "2", + "7" + ], + [ + "2", + "3aa85881-1185-642f-c44b-41ad2275d2", + ")ud275d2", + "hum-almsh022km2(13)ud2", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(13)1", + "81.30000305175781", + null, + null, + "pass", + "2", + "", + "", + "", + "26", + "96", + "6", + "28", + "3", + "2", + "7" + ], + [ + "3", + "fa418804-6c4f-4c90-d78f-84d7df54d3", + ")ud354d3", + "ene-wash017okm2(82)ud3", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(82)1", + "92.19999694824219", + null, + null, + "pass", + "3", + "", + "", + "", + "2", + "91", + "9", + "28", + "3", + "2", + "7" + ], + [ + "4", + "6fdacbfc-7e0b-473b-444f-85b7650267", + ")u670267", + "ebo-gppm010okm2(1b)u67", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(1b)1", + "93.80000305175781", + null, + null, + "pass", + "4", + "", + "", + "", + "7", + "153", + "9", + "28", + "3", + "2", + "7" + ] + ], + "shape": { + "columns": 22, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ult_uuid_037record_idult_sample_namestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_at...equipmentraw_data_urlupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
0421617a5-e50e-9642-2974-b3275fe822)u22e822hum-almhu023km2(15)u22rt vacuum sealedult26kh1ult26kh(15)188.599998NaTNaT...880628327
17e7919c2-5db4-6bef-75e2-7e51321200)u001200hum-almhu023km2(15)u00rt vacuum sealedult26kh1ult26kh(15)1NaNNaTNaT...880628327
23aa85881-1185-642f-c44b-41ad2275d2)ud275d2hum-almsh022km2(13)ud2rt vacuum sealedult26kh1ult26kh(13)181.300003NaTNaT...2696628327
3fa418804-6c4f-4c90-d78f-84d7df54d3)ud354d3ene-wash017okm2(82)ud3rt vacuum sealedult26kh1ult26kh(82)192.199997NaTNaT...291928327
46fdacbfc-7e0b-473b-444f-85b7650267)u670267ebo-gppm010okm2(1b)u67rt vacuum sealedult26kh1ult26kh(1b)193.800003NaTNaT...7153928327
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " ult_uuid_037 record_id ult_sample_name \\\n", + "0 421617a5-e50e-9642-2974-b3275fe822 )u22e822 hum-almhu023km2(15)u22 \n", + "1 7e7919c2-5db4-6bef-75e2-7e51321200 )u001200 hum-almhu023km2(15)u00 \n", + "2 3aa85881-1185-642f-c44b-41ad2275d2 )ud275d2 hum-almsh022km2(13)ud2 \n", + "3 fa418804-6c4f-4c90-d78f-84d7df54d3 )ud354d3 ene-wash017okm2(82)ud3 \n", + "4 6fdacbfc-7e0b-473b-444f-85b7650267 )u670267 ebo-gppm010okm2(1b)u67 \n", + "\n", + " storage_cond exper_abbrev repl_no repl_id value created_at \\\n", + "0 rt vacuum sealed ult26kh 1 ult26kh(15)1 88.599998 NaT \n", + "1 rt vacuum sealed ult26kh 1 ult26kh(15)1 NaN NaT \n", + "2 rt vacuum sealed ult26kh 1 ult26kh(13)1 81.300003 NaT \n", + "3 rt vacuum sealed ult26kh 1 ult26kh(82)1 92.199997 NaT \n", + "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", + "\n", + " updated_at ... equipment raw_data_url upload_status resource_id \\\n", + "0 NaT ... 8 \n", + "1 NaT ... 8 \n", + "2 NaT ... 26 \n", + "3 NaT ... 2 \n", + "4 NaT ... 7 \n", + "\n", + " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", + "0 80 6 28 3 \n", + "1 80 6 28 3 \n", + "2 96 6 28 3 \n", + "3 91 9 28 3 \n", + "4 153 9 28 3 \n", + "\n", + " analyst_email_id analysis_type_id \n", + "0 2 7 \n", + "1 2 7 \n", + "2 2 7 \n", + "3 2 7 \n", + "4 2 7 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 3 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "31e4fb08-8acb-41fc-bdec-4f79aea24d5f", + "rows": [ + [ + "0", + "3ee2993d-86e3-1f16-c7ea-f8d555e114", + "(85)e114", + "rt vacuum sealed", + "cmp04xk", + "1", + "cmp04xk(85)1", + "14.15999984741211", + "2025-01-23 09:00:01", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "1", + "46878ef9-1226-22a0-d5d8-cf65e241cb", + "(85)41cb", + "rt vacuum sealed", + "cmp04xk", + "2", + "cmp04xk(85)2", + "14.180000305175781", + "2025-01-23 09:00:16", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "2", + "76a7a2f4-c4e4-e60f-1187-dec6e02246", + "(85)2246", + "rt vacuum sealed", + "cmp04xk", + "3", + "cmp04xk(85)3", + "14.119999885559082", + "2025-01-23 09:00:31", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "3", + "7a136832-286b-07cb-62de-acf52f9311", + "(85)9311", + "rt vacuum sealed", + "cmp04xk", + "1", + "cmp04xk(85)1", + "15.739999771118164", + "2025-01-23 09:00:46", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "14", + "2", + "1", + "3" + ], + [ + "4", + "b709ecee-f9a6-a55d-a59e-93b7b863d7", + "(85)63d7", + "rt vacuum sealed", + "cmp04xk", + "2", + "cmp04xk(85)2", + "15.75", + "2025-01-23 09:01:01", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "14", + "2", + "1", + "3" + ] + ], + "shape": { + "columns": 21, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cmp_uuid_033record_idstorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_result...equipmentraw_data_urlupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
03ee2993d-86e3-1f16-c7ea-f8d555e114(85)e114rt vacuum sealedcmp04xk1cmp04xk(85)114.162025-01-23 09:00:01NaTpass...ready2492812213
146878ef9-1226-22a0-d5d8-cf65e241cb(85)41cbrt vacuum sealedcmp04xk2cmp04xk(85)214.182025-01-23 09:00:16NaTpass...ready2492812213
276a7a2f4-c4e4-e60f-1187-dec6e02246(85)2246rt vacuum sealedcmp04xk3cmp04xk(85)314.122025-01-23 09:00:31NaTpass...ready2492812213
37a136832-286b-07cb-62de-acf52f9311(85)9311rt vacuum sealedcmp04xk1cmp04xk(85)115.742025-01-23 09:00:46NaTpass...ready2492814213
4b709ecee-f9a6-a55d-a59e-93b7b863d7(85)63d7rt vacuum sealedcmp04xk2cmp04xk(85)215.752025-01-23 09:01:01NaTpass...ready2492814213
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " cmp_uuid_033 record_id storage_cond \\\n", + "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 rt vacuum sealed \n", + "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb rt vacuum sealed \n", + "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 rt vacuum sealed \n", + "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 rt vacuum sealed \n", + "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 rt vacuum sealed \n", + "\n", + " exper_abbrev repl_no repl_id value created_at updated_at \\\n", + "0 cmp04xk 1 cmp04xk(85)1 14.16 2025-01-23 09:00:01 NaT \n", + "1 cmp04xk 2 cmp04xk(85)2 14.18 2025-01-23 09:00:16 NaT \n", + "2 cmp04xk 3 cmp04xk(85)3 14.12 2025-01-23 09:00:31 NaT \n", + "3 cmp04xk 1 cmp04xk(85)1 15.74 2025-01-23 09:00:46 NaT \n", + "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", + "\n", + " qc_result ... equipment raw_data_url upload_status resource_id \\\n", + "0 pass ... ready 24 \n", + "1 pass ... ready 24 \n", + "2 pass ... ready 24 \n", + "3 pass ... ready 24 \n", + "4 pass ... ready 24 \n", + "\n", + " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", + "0 92 8 12 2 \n", + "1 92 8 12 2 \n", + "2 92 8 12 2 \n", + "3 92 8 14 2 \n", + "4 92 8 14 2 \n", + "\n", + " analyst_email_id analysis_type_id \n", + "0 1 3 \n", + "1 1 3 \n", + "2 1 3 \n", + "3 1 3 \n", + "4 1 3 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- 1. Extraction ---\n", + "# In a real Prefect flow, each extraction would be a separate task.\n", + "logger.info('Starting data extraction...')\n", + "prox_df = proximate.extract(project_root=project_root)\n", + "ult_df = ultimate.extract(project_root=project_root)\n", + "cmp_df = cmpana.extract(project_root=project_root)\n", + "dataframes = [prox_df, ult_df, cmp_df]\n", + "logger.info('Data extraction complete.')\n", + "\n", + "# --- 2. Cleaning ---\n", + "# This list comprehension applies the cleaning function to each extracted dataframe.\n", + "logger.info('Starting data cleaning...')\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes if df is not None]\n", + "logger.info('Data cleaning complete.')\n", + "\n", + "# --- 3. Normalization ---\n", + "# This dictionary defines the columns to be normalized. \n", + "# The key is the column name in the DataFrame.\n", + "# The value is a tuple containing the corresponding SQLAlchemy model and the name of the attribute on the model to match against.\n", + "NORMALIZE_COLUMNS = {\n", + " 'resource': (Resource, 'name'),\n", + " 'prepared_sample': (PreparedSample, 'name'),\n", + " 'preparation_method': (PreparationMethod, 'name'),\n", + " 'parameter': (Parameter, 'name'),\n", + " 'unit': (Unit, 'name'),\n", + " 'analyst_email': (Contact, 'email'),\n", + " 'analysis_type': (AnalysisType, 'name'),\n", + " 'primary_ag_product': (PrimaryAgProduct, 'name')\n", + "}\n", + "\n", + "logger.info('Starting data normalization...')\n", + "normalized_dataframes = normalize_dataframes(clean_dataframes, NORMALIZE_COLUMNS)\n", + "logger.info('Data normalization complete.')\n", + "\n", + "# --- 4. Display Results ---\n", + "logger.info('Displaying results of normalization...')\n", + "for i, df in enumerate(normalized_dataframes):\n", + " print(f'--- Normalized DataFrame {i+1} ---')\n", + " display(df.head())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deployment Plan\n", + "\n", + "The code in this notebook will be transitioned to the main ETL pipeline by following these steps:\n", + "\n", + "1. **Function Migration**: The `clean_the_gsheets` and `normalize_dataframes` functions will be moved to a new utility module, for example, `src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_utils.py`. Each function will be decorated with `@task` from Prefect to turn it into a reusable pipeline component.\n", + "2. **Flow Creation**: A new Prefect flow will be created in the `src/ca_biositing/pipeline/ca_biositing/pipeline/flows/` directory (e.g., `master_extraction_flow.py`). This flow will orchestrate the entire ETL process for a given data source.\n", + "3. **Task Integration**: The new flow will be composed of individual tasks. It will call the existing extraction tasks (`proximate.extract`, etc.), and then pass the results to the new cleaning and normalization tasks from `etl_utils.py`.\n", + "4. **Logging**: The `logging` module will be replaced with `get_run_logger()` from Prefect within the tasks to ensure logs are captured by the Prefect UI.\n", + "5. **Configuration**: The `NORMALIZE_COLUMNS` dictionary will be moved to a configuration file or defined within the relevant flow to make it easier to manage and modify without changing the code.\n", + "6. **Testing**: Unit tests will be written for the new utility functions in `etl_utils.py`. An integration test will be created for the new Prefect flow to ensure all the tasks work together correctly.\n", + "7. **Deployment**: Once the flow is complete and tested, it will be deployed to the Prefect server using the `pixi run deploy` command, making it available to be run on a schedule or manually via the UI." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Pixi Default Environment", + "language": "python", + "name": "pixi-default" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb index 9d27799e..bfb3c8c4 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -2,62 +2,130 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "from IPython.display import display\n", + "\n", + "# --- Robustly find the project root ---\n", + "# The project root is the directory containing the 'pixi.toml' file.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path): # Stop at the filesystem root\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# --- Add project root to sys.path ---\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " print(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " print(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import the module ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " print(\"Successfully imported all module.\")\n", + "except ImportError as e:\n", + " print(f\"Failed to import modules: {e}\")\n", + " print(f\"\\nFull sys.path: {sys.path}\")\n", + "\n", + "# --- Run the extraction ---\n", + "if 'proximate' in locals():\n", + " try:\n", + " # Pass the project_root to the extract function\n", + " df = proximate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted proximate data.\")\n", + " display(df.head())\n", + " else:\n", + " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", + "\n", + "if 'ultimate' in locals():\n", + " try:\n", + " df2 = ultimate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted Ultimate data.\")\n", + " display(df2.head())\n", + " else:\n", + " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during extraction: {e}\")\n", + "\n", + "if 'cmpana' in locals():\n", + " try:\n", + " df3 = cmpana.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted CmpAna data.\")\n", + " display(df3.head())\n", + " else:\n", + " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", + " finally:\n", + " print(\"\\nCmp Ana extraction process completed.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " # 1. Clean names and drop rows\n", + " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + "\n", + " # 2. Coerce types (using errors='coerce' handles messy string data)\n", + " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", + " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Dates\n", + " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", + " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert remaining objects to best possible types (like strings)\n", + " df = df.convert_dtypes()\n", + "\n", + " # 5. Convert all string data to lowercase\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "\n", + " # 6. Convert empty strings to NaN\n", + " df.replace(\"\", np.nan, inplace=True)\n", + "\n", + " return df # Return the FULL dataframe, not just .head()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", - "Successfully imported all module.\n" - ] - }, - { - "data": { - "text/html": [ - "
20:54:42.753 | INFO    | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:42.753 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.230 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.230 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.234 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.234 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted proximate data.\n" + "Connected to database.\n" ] }, { @@ -70,228 +138,182 @@ "type": "integer" }, { - "name": "Prox_UUID_031", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Source_codename", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Parameter", - "rawType": "object", - "type": "string" - }, - { - "name": "Value", - "rawType": "object", - "type": "string" - }, - { - "name": "Unit", - "rawType": "object", - "type": "string" - }, - { - "name": "Created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "Updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "QC_result", - "rawType": "object", - "type": "string" + "name": "id", + "rawType": "int64", + "type": "integer" }, { - "name": "Upload_status", + "name": "name", "rawType": "object", "type": "string" }, { - "name": "Note", + "name": "note", "rawType": "object", - "type": "string" + "type": "unknown" }, { - "name": "Analysis_type", + "name": "description", "rawType": "object", - "type": "string" + "type": "unknown" }, { - "name": "Analyst_email", + "name": "uri", "rawType": "object", - "type": "string" + "type": "unknown" } ], - "ref": "b6c1f313-06e4-4dba-8a9e-6a1dc878820f", + "ref": "f594ce12-31fc-400e-807f-9ae024f154ba", "rows": [ [ "0", - "D7965110-407F-E356-D41D-B3B9A2B7B7", - "(73)B7B7", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "1", - "Prox01xk(73)1", - "Moisture", - "61.85", - "% total weight", - "2024-10-02 10:31:01", - "", - "Pass", - "not ready", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Tomatoes for processing", + null, + null, + null ], [ "1", - "C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8", - "(73)9BA8", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "2", - "Prox01xk(73)2", - "Moisture", - "63.21", - "% total weight", - "2024-10-02 10:31:31", - "", - "Pass", - "ready", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Grapes", + null, + null, + null ], [ "2", - "DF304D5D-3A85-4881-7142-6D4E5F957D", - "(73)957D", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "3", - "Prox01xk(73)3", - "Moisture", - "63.27", - "% total weight", - "2024-10-02 10:32:01", - "", - "Pass", - "imported", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Almonds", + null, + null, + null ], [ "3", - "01C6C5BE-CEA6-54AF-3924-B0BAD69335", - "(73)9335", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", - "1", - "Prox01xk(73)1", - "Ash", - "0.69", - "% total weight", - "2024-10-03 10:31:01", - "", - "Pass", - "import failed", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "4", + "Walnuts", + null, + null, + null ], [ "4", - "126745C7-DD41-2F6D-0DC5-28DBCA415F", - "(73)415F", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", - "2", - "Prox01xk(73)2", - "Ash", - "0.89", - "% total weight", - "2024-10-03 10:31:31", - "", - "Pass", - "", - "", - "Proximate analysis", - "xkang2@lbl.gov" - ] - ], - "shape": { - "columns": 20, - "rows": 5 - } + "5", + "Sweet potatoes", + null, + null, + null + ], + [ + "5", + "6", + "Algae", + null, + null, + null + ], + [ + "6", + "7", + "Olives - processing", + null, + null, + null + ], + [ + "7", + "8", + "Corn - all", + null, + null, + null + ], + [ + "8", + "9", + "Hay - alfalfa", + null, + null, + null + ], + [ + "9", + "10", + "Silage - wheat", + null, + null, + null + ], + [ + "10", + "11", + "Rice", + null, + null, + null + ], + [ + "11", + "12", + "Peaches", + null, + null, + null + ], + [ + "12", + "13", + "Potatoes", + null, + null, + null + ], + [ + "13", + "14", + "Cucumbers and pickles", + null, + null, + null + ], + [ + "14", + "15", + "Pistachios", + null, + null, + null + ], + [ + "15", + "16", + "Cotton", + null, + null, + null + ], + [ + "16", + "17", + "Olives - market", + null, + null, + null + ], + [ + "17", + "18", + "", + null, + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 18 + } }, "text/html": [ "
\n", @@ -312,3436 +334,527 @@ " \n", " \n", " \n", - " Prox_UUID_031\n", - " Record_ID\n", - " Source_codename\n", - " Prepared_sample\n", - " Resource\n", - " Preparation_method\n", - " Storage_cond\n", - " Exper_abbrev\n", - " Repl_no\n", - " Repl_ID\n", - " Parameter\n", - " Value\n", - " Unit\n", - " Created_at\n", - " Updated_at\n", - " QC_result\n", - " Upload_status\n", - " Note\n", - " Analysis_type\n", - " Analyst_email\n", + " id\n", + " name\n", + " note\n", + " description\n", + " uri\n", " \n", " \n", " \n", " \n", " 0\n", - " D7965110-407F-E356-D41D-B3B9A2B7B7\n", - " (73)B7B7\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 1\n", - " Prox01xk(73)1\n", - " Moisture\n", - " 61.85\n", - " % total weight\n", - " 2024-10-02 10:31:01\n", - " \n", - " Pass\n", - " not ready\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Tomatoes for processing\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 1\n", - " C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8\n", - " (73)9BA8\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 2\n", - " Prox01xk(73)2\n", - " Moisture\n", - " 63.21\n", - " % total weight\n", - " 2024-10-02 10:31:31\n", - " \n", - " Pass\n", - " ready\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Grapes\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 2\n", - " DF304D5D-3A85-4881-7142-6D4E5F957D\n", - " (73)957D\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 3\n", - " Prox01xk(73)3\n", - " Moisture\n", - " 63.27\n", - " % total weight\n", - " 2024-10-02 10:32:01\n", - " \n", - " Pass\n", - " imported\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Almonds\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 3\n", - " 01C6C5BE-CEA6-54AF-3924-B0BAD69335\n", - " (73)9335\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", - " 1\n", - " Prox01xk(73)1\n", - " Ash\n", - " 0.69\n", - " % total weight\n", - " 2024-10-03 10:31:01\n", - " \n", - " Pass\n", - " import failed\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " 4\n", + " Walnuts\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 4\n", - " 126745C7-DD41-2F6D-0DC5-28DBCA415F\n", - " (73)415F\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", - " 2\n", - " Prox01xk(73)2\n", - " Ash\n", - " 0.89\n", - " % total weight\n", - " 2024-10-03 10:31:31\n", - " \n", - " Pass\n", - " \n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " 5\n", + " Sweet potatoes\n", + " None\n", + " None\n", + " None\n", " \n", - " \n", - "\n", - "
" - ], - "text/plain": [ - " Prox_UUID_031 Record_ID Source_codename \\\n", - "0 D7965110-407F-E356-D41D-B3B9A2B7B7 (73)B7B7 Oakleaf \n", - "1 C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8 (73)9BA8 Oakleaf \n", - "2 DF304D5D-3A85-4881-7142-6D4E5F957D (73)957D Oakleaf \n", - "3 01C6C5BE-CEA6-54AF-3924-B0BAD69335 (73)9335 Oakleaf \n", - "4 126745C7-DD41-2F6D-0DC5-28DBCA415F (73)415F Oakleaf \n", - "\n", - " Prepared_sample Resource Preparation_method Storage_cond \\\n", - "0 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "1 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "2 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "3 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "4 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "\n", - " Exper_abbrev Repl_no Repl_ID Parameter Value Unit \\\n", - "0 Prox01xk 1 Prox01xk(73)1 Moisture 61.85 % total weight \n", - "1 Prox01xk 2 Prox01xk(73)2 Moisture 63.21 % total weight \n", - "2 Prox01xk 3 Prox01xk(73)3 Moisture 63.27 % total weight \n", - "3 Prox01xk 1 Prox01xk(73)1 Ash 0.69 % total weight \n", - "4 Prox01xk 2 Prox01xk(73)2 Ash 0.89 % total weight \n", - "\n", - " Created_at Updated_at QC_result Upload_status Note \\\n", - "0 2024-10-02 10:31:01 Pass not ready \n", - "1 2024-10-02 10:31:31 Pass ready \n", - "2 2024-10-02 10:32:01 Pass imported \n", - "3 2024-10-03 10:31:01 Pass import failed \n", - "4 2024-10-03 10:31:31 Pass \n", - "\n", - " Analysis_type Analyst_email \n", - "0 Proximate analysis xkang2@lbl.gov \n", - "1 Proximate analysis xkang2@lbl.gov \n", - "2 Proximate analysis xkang2@lbl.gov \n", - "3 Proximate analysis xkang2@lbl.gov \n", - "4 Proximate analysis xkang2@lbl.gov " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.272 | INFO    | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.272 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.593 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.593 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.595 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.595 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted Ultimate data.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Ult_UUID_037", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Ult_sample_name", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, + " \n", + " 5\n", + " 6\n", + " Algae\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 6\n", + " 7\n", + " Olives - processing\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 7\n", + " 8\n", + " Corn - all\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 8\n", + " 9\n", + " Hay - alfalfa\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 9\n", + " 10\n", + " Silage - wheat\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 10\n", + " 11\n", + " Rice\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 11\n", + " 12\n", + " Peaches\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 12\n", + " 13\n", + " Potatoes\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 13\n", + " 14\n", + " Cucumbers and pickles\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 14\n", + " 15\n", + " Pistachios\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 15\n", + " 16\n", + " Cotton\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 16\n", + " 17\n", + " Olives - market\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 17\n", + " 18\n", + " \n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " id name note description uri\n", + "0 1 Tomatoes for processing None None None\n", + "1 2 Grapes None None None\n", + "2 3 Almonds None None None\n", + "3 4 Walnuts None None None\n", + "4 5 Sweet potatoes None None None\n", + "5 6 Algae None None None\n", + "6 7 Olives - processing None None None\n", + "7 8 Corn - all None None None\n", + "8 9 Hay - alfalfa None None None\n", + "9 10 Silage - wheat None None None\n", + "10 11 Rice None None None\n", + "11 12 Peaches None None None\n", + "12 13 Potatoes None None None\n", + "13 14 Cucumbers and pickles None None None\n", + "14 15 Pistachios None None None\n", + "15 16 Cotton None None None\n", + "16 17 Olives - market None None None\n", + "17 18 None None None" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sqlmodel import Session, select, create_engine\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "#This module queries the db via the ORM\n", + "\n", + "# Database Connection\n", + "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", + "engine = create_engine(DATABASE_URL)\n", + "print(f\"Connected to database.\")\n", + "\n", + "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", + "\n", + "#reorders columns so id and name are first\n", + "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", + "\n", + "primary_ag_product = primary_ag_product[[*cols]]\n", + "\n", + "primary_ag_product\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "resource", + "rawType": "object", + "type": "string" + }, { - "name": "Parameter", + "name": "preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", "rawType": "object", "type": "string" }, { - "name": "Value", + "name": "exper_abbrev", "rawType": "object", "type": "string" }, { - "name": "Unit", + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", "rawType": "object", "type": "string" }, { - "name": "Created_at", + "name": "parameter", "rawType": "object", "type": "string" }, { - "name": "Updated_at", + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "unit", "rawType": "object", "type": "string" }, { - "name": "QC_result", + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", "rawType": "object", "type": "string" }, { - "name": "Note", + "name": "note", "rawType": "object", "type": "string" }, { - "name": "Analysis_type", + "name": "analysis_type", "rawType": "object", "type": "string" }, { - "name": "Equipment", + "name": "equipment", "rawType": "object", "type": "string" }, { - "name": "Raw_data_URL", + "name": "raw_data_url", "rawType": "object", "type": "string" }, { - "name": "Analyst_email", + "name": "analyst_email", "rawType": "object", "type": "string" }, { - "name": "Upload_status", + "name": "upload_status", "rawType": "object", "type": "string" } ], - "ref": "61714e59-0541-4b74-9d38-a37794cbd34d", - "rows": [ - [ - "0", - "421617A5-E50E-9642-2974-B3275FE822", - ")U22E822", - "Hum-AlmHu023KM2(15)U22", - "Hum-AlmHu023KM2(15)", - "Almond Hulls", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(15)1", - "DM", - "8.86E+01", - "pc", - "", - "", - "Pass", - "1", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "1", - "7E7919C2-5DB4-6BEF-75E2-7E51321200", - ")U001200", - "Hum-AlmHu023KM2(15)U00", - "Hum-AlmHu023KM2(15)", - "Almond Hulls", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(15)1", - "DM", - "", - "pc", - "", - "", - "Fail", - " 1 dup", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "2", - "3AA85881-1185-642F-C44B-41AD2275D2", - ")UD275D2", - "Hum-AlmSh022KM2(13)UD2", - "Hum-AlmSh022KM2(13)", - "Almond Shells", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(13)1", - "DM", - "8.13E+01", - "pc", - "", - "", - "Pass", - "2", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "3", - "FA418804-6C4F-4C90-D78F-84D7DF54D3", - ")UD354D3", - "Ene-WaSh017OKM2(82)UD3", - "Ene-WaSh017OKM2(82)", - "Walnut Shells", - "Oven Dry + Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(82)1", - "DM", - "9.22E+01", - "pc", - "", - "", - "Pass", - "3", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "4", - "6FDACBFC-7E0B-473B-444F-85B7650267", - ")U670267", - "Ebo-GpPm010OKM2(1B)U67", - "Ebo-GpPm010OKM2(1B)", - "Grape pomace", - "Oven Dry + Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(1B)1", - "DM", - "9.38E+01", - "pc", - "", - "", - "Pass", - "4", - "Ultimate analysis", - "", - "", - "", - "" - ] - ], - "shape": { - "columns": 22, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Ult_UUID_037Record_IDUlt_sample_namePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_ID...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
0421617A5-E50E-9642-2974-B3275FE822)U22E822Hum-AlmHu023KM2(15)U22Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcPass1Ultimate analysis
17E7919C2-5DB4-6BEF-75E2-7E51321200)U001200Hum-AlmHu023KM2(15)U00Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcFail1 dupUltimate analysis
23AA85881-1185-642F-C44B-41AD2275D2)UD275D2Hum-AlmSh022KM2(13)UD2Hum-AlmSh022KM2(13)Almond ShellsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(13)1...pcPass2Ultimate analysis
3FA418804-6C4F-4C90-D78F-84D7DF54D3)UD354D3Ene-WaSh017OKM2(82)UD3Ene-WaSh017OKM2(82)Walnut ShellsOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(82)1...pcPass3Ultimate analysis
46FDACBFC-7E0B-473B-444F-85B7650267)U670267Ebo-GpPm010OKM2(1B)U67Ebo-GpPm010OKM2(1B)Grape pomaceOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(1B)1...pcPass4Ultimate analysis
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " Ult_UUID_037 Record_ID Ult_sample_name \\\n", - "0 421617A5-E50E-9642-2974-B3275FE822 )U22E822 Hum-AlmHu023KM2(15)U22 \n", - "1 7E7919C2-5DB4-6BEF-75E2-7E51321200 )U001200 Hum-AlmHu023KM2(15)U00 \n", - "2 3AA85881-1185-642F-C44B-41AD2275D2 )UD275D2 Hum-AlmSh022KM2(13)UD2 \n", - "3 FA418804-6C4F-4C90-D78F-84D7DF54D3 )UD354D3 Ene-WaSh017OKM2(82)UD3 \n", - "4 6FDACBFC-7E0B-473B-444F-85B7650267 )U670267 Ebo-GpPm010OKM2(1B)U67 \n", - "\n", - " Prepared_sample Resource Preparation_method \\\n", - "0 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", - "1 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", - "2 Hum-AlmSh022KM2(13) Almond Shells Knife Mill (2mm) \n", - "3 Ene-WaSh017OKM2(82) Walnut Shells Oven Dry + Knife Mill (2mm) \n", - "4 Ebo-GpPm010OKM2(1B) Grape pomace Oven Dry + Knife Mill (2mm) \n", - "\n", - " Storage_cond Exper_abbrev Repl_no Repl_ID ... Unit Created_at \\\n", - "0 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", - "1 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", - "2 RT vacuum sealed Ult26kh 1 Ult26kh(13)1 ... pc \n", - "3 RT vacuum sealed Ult26kh 1 Ult26kh(82)1 ... pc \n", - "4 RT vacuum sealed Ult26kh 1 Ult26kh(1B)1 ... pc \n", - "\n", - " Updated_at QC_result Note Analysis_type Equipment Raw_data_URL \\\n", - "0 Pass 1 Ultimate analysis \n", - "1 Fail 1 dup Ultimate analysis \n", - "2 Pass 2 Ultimate analysis \n", - "3 Pass 3 Ultimate analysis \n", - "4 Pass 4 Ultimate analysis \n", - "\n", - " Analyst_email Upload_status \n", - "0 \n", - "1 \n", - "2 \n", - "3 \n", - "4 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.631 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.631 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:47.211 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:47.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:47.214 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:47.214 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted CmpAna data.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Cmp_UUID_033", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Parameter", - "rawType": "object", - "type": "string" - }, - { - "name": "Value", - "rawType": "object", - "type": "string" - }, - { - "name": "Unit", - "rawType": "object", - "type": "string" - }, - { - "name": "Created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "Updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "QC_result", - "rawType": "object", - "type": "string" - }, - { - "name": "Note", - "rawType": "object", - "type": "string" - }, - { - "name": "Analysis_type", - "rawType": "object", - "type": "string" - }, - { - "name": "Equipment", - "rawType": "object", - "type": "string" - }, - { - "name": "Raw_data_URL", - "rawType": "object", - "type": "string" - }, - { - "name": "Analyst_email", - "rawType": "object", - "type": "string" - }, - { - "name": "Upload_status", - "rawType": "object", - "type": "string" - } - ], - "ref": "2dd03f9d-30e4-4e23-af07-409bfdca5047", - "rows": [ - [ - "0", - "3EE2993D-86E3-1F16-C7EA-F8D555E114", - "(85)E114", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "Glucan", - "14.16", - "% dry weight", - "1/23/2025 9:00:01", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "1", - "46878EF9-1226-22A0-D5D8-CF65E241CB", - "(85)41CB", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "Glucan", - "14.18", - "% dry weight", - "1/23/2025 9:00:16", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "2", - "76A7A2F4-C4E4-E60F-1187-DEC6E02246", - "(85)2246", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "Glucan", - "14.12", - "% dry weight", - "1/23/2025 9:00:31", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "3", - "7A136832-286B-07CB-62DE-ACF52F9311", - "(85)9311", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "Glucose", - "15.74", - "% dry weight", - "1/23/2025 9:00:46", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "4", - "B709ECEE-F9A6-A55D-A59E-93B7B863D7", - "(85)63D7", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "Glucose", - "15.75", - "% dry weight", - "1/23/2025 9:01:01", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ] - ], - "shape": { - "columns": 21, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Cmp_UUID_033Record_IDPrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameter...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucan...% dry weight1/23/2025 9:00:01passChemical compositionxkang2@lbl.govready
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucan...% dry weight1/23/2025 9:00:16passChemical compositionxkang2@lbl.govready
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)3Glucan...% dry weight1/23/2025 9:00:31passChemical compositionxkang2@lbl.govready
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucose...% dry weight1/23/2025 9:00:46passChemical compositionxkang2@lbl.govready
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucose...% dry weight1/23/2025 9:01:01passChemical compositionxkang2@lbl.govready
\n", - "

5 rows × 21 columns

\n", - "
" - ], - "text/plain": [ - " Cmp_UUID_033 Record_ID Prepared_sample \\\n", - "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", - "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", - "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", - "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", - "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", - "\n", - " Resource Preparation_method Storage_cond Exper_abbrev Repl_no \\\n", - "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", - "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", - "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 3 \n", - "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", - "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", - "\n", - " Repl_ID Parameter ... Unit Created_at Updated_at \\\n", - "0 Cmp04xk(85)1 Glucan ... % dry weight 1/23/2025 9:00:01 \n", - "1 Cmp04xk(85)2 Glucan ... % dry weight 1/23/2025 9:00:16 \n", - "2 Cmp04xk(85)3 Glucan ... % dry weight 1/23/2025 9:00:31 \n", - "3 Cmp04xk(85)1 Glucose ... % dry weight 1/23/2025 9:00:46 \n", - "4 Cmp04xk(85)2 Glucose ... % dry weight 1/23/2025 9:01:01 \n", - "\n", - " QC_result Note Analysis_type Equipment Raw_data_URL Analyst_email \\\n", - "0 pass Chemical composition xkang2@lbl.gov \n", - "1 pass Chemical composition xkang2@lbl.gov \n", - "2 pass Chemical composition xkang2@lbl.gov \n", - "3 pass Chemical composition xkang2@lbl.gov \n", - "4 pass Chemical composition xkang2@lbl.gov \n", - "\n", - " Upload_status \n", - "0 ready \n", - "1 ready \n", - "2 ready \n", - "3 ready \n", - "4 ready \n", - "\n", - "[5 rows x 21 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Cmp Ana extraction process completed.\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import pandas as pd\n", - "import numpy as np\n", - "import janitor as jn\n", - "from IPython.display import display\n", - "\n", - "# --- Robustly find the project root ---\n", - "# The project root is the directory containing the 'pixi.toml' file.\n", - "path = os.getcwd()\n", - "project_root = None\n", - "while path != os.path.dirname(path): # Stop at the filesystem root\n", - " if 'pixi.toml' in os.listdir(path):\n", - " project_root = path\n", - " break\n", - " path = os.path.dirname(path)\n", - "\n", - "if not project_root:\n", - " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", - "\n", - "# --- Add project root to sys.path ---\n", - "if project_root not in sys.path:\n", - " sys.path.insert(0, project_root)\n", - " print(f\"Added project root '{project_root}' to sys.path\")\n", - "else:\n", - " print(f\"Project root '{project_root}' is already in sys.path\")\n", - "\n", - "# --- Import the module ---\n", - "try:\n", - " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", - " print(\"Successfully imported all module.\")\n", - "except ImportError as e:\n", - " print(f\"Failed to import modules: {e}\")\n", - " print(f\"\\nFull sys.path: {sys.path}\")\n", - "\n", - "# --- Run the extraction ---\n", - "if 'proximate' in locals():\n", - " try:\n", - " # Pass the project_root to the extract function\n", - " df = proximate.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted proximate data.\")\n", - " display(df.head())\n", - " else:\n", - " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", - "\n", - "if 'ultimate' in locals():\n", - " try:\n", - " df2 = ultimate.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted Ultimate data.\")\n", - " display(df2.head())\n", - " else:\n", - " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during extraction: {e}\")\n", - "\n", - "if 'cmpana' in locals():\n", - " try:\n", - " df3 = cmpana.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted CmpAna data.\")\n", - " display(df3.head())\n", - " else:\n", - " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", - " finally:\n", - " print(\"\\nCmp Ana extraction process completed.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def clean_the_gsheets(df):\n", - " # 1. Clean names and drop rows\n", - " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", - "\n", - " # 2. Coerce types (using errors='coerce' handles messy string data)\n", - " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", - " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", - "\n", - " # 3. Dates\n", - " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", - " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", - "\n", - " # 4. Convert remaining objects to best possible types (like strings)\n", - " df = df.convert_dtypes()\n", - " \n", - " return df # Return the FULL dataframe, not just .head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframes = [df, df2, df3]\n", - "\n", - "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", - "\n", - "clean_dataframes[2].head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "summary_stats = clean_dataframes[0].\\\n", - " groupby(['resource', 'parameter'])['value'].\\\n", - " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", - "\n", - "summary_stats" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", - " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", - " agg({'value': 'mean'}).\\\n", - " query('value > 30').\\\n", - " sort_values(by='value', ascending=False).\\\n", - " round({'value': 1})\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", - "\n", - "def is_it_volatile_solids(df):\n", - " df['check'] = \"VS\"\n", - "\n", - " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", - " return df\n", - "\n", - "is_it_volatile_solids(df)\n", - "\n", - "df[['check', 'parameter']]\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", - "def sqrtvalue(df):\n", - " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", - " return df\n", - "\n", - "#List comprehension to apply sqrtvalue to each DataFrame\n", - "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", - "\n", - "# Display the head of the third DataFrame\n", - "clean_rooted_df[2].head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cmpana_raw = cmpana.extract(project_root=project_root)\n", - "\n", - "cmpana_raw.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected to database.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "name", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "unknown" - }, - { - "name": "description", - "rawType": "object", - "type": "unknown" - }, - { - "name": "uri", - "rawType": "object", - "type": "unknown" - } - ], - "ref": "0dabe9e2-728a-486d-9dd1-c1edd094e9c2", - "rows": [ - [ - "0", - "1", - "Tomatoes for processing", - null, - null, - null - ], - [ - "1", - "2", - "Grapes", - null, - null, - null - ], - [ - "2", - "3", - "Almonds", - null, - null, - null - ], - [ - "3", - "4", - "Walnuts", - null, - null, - null - ], - [ - "4", - "5", - "Sweet potatoes", - null, - null, - null - ], - [ - "5", - "6", - "Algae", - null, - null, - null - ], - [ - "6", - "7", - "Olives - processing", - null, - null, - null - ], - [ - "7", - "8", - "Corn - all", - null, - null, - null - ], - [ - "8", - "9", - "Hay - alfalfa", - null, - null, - null - ], - [ - "9", - "10", - "Silage - wheat", - null, - null, - null - ], - [ - "10", - "11", - "Rice", - null, - null, - null - ], - [ - "11", - "12", - "Peaches", - null, - null, - null - ], - [ - "12", - "13", - "Potatoes", - null, - null, - null - ], - [ - "13", - "14", - "Cucumbers and pickles", - null, - null, - null - ], - [ - "14", - "15", - "Pistachios", - null, - null, - null - ], - [ - "15", - "16", - "Cotton", - null, - null, - null - ], - [ - "16", - "17", - "Olives - market", - null, - null, - null - ], - [ - "17", - "18", - "", - null, - null, - null - ] - ], - "shape": { - "columns": 5, - "rows": 18 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamenotedescriptionuri
01Tomatoes for processingNoneNoneNone
12GrapesNoneNoneNone
23AlmondsNoneNoneNone
34WalnutsNoneNoneNone
45Sweet potatoesNoneNoneNone
56AlgaeNoneNoneNone
67Olives - processingNoneNoneNone
78Corn - allNoneNoneNone
89Hay - alfalfaNoneNoneNone
910Silage - wheatNoneNoneNone
1011RiceNoneNoneNone
1112PeachesNoneNoneNone
1213PotatoesNoneNoneNone
1314Cucumbers and picklesNoneNoneNone
1415PistachiosNoneNoneNone
1516CottonNoneNoneNone
1617Olives - marketNoneNoneNone
1718NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " id name note description uri\n", - "0 1 Tomatoes for processing None None None\n", - "1 2 Grapes None None None\n", - "2 3 Almonds None None None\n", - "3 4 Walnuts None None None\n", - "4 5 Sweet potatoes None None None\n", - "5 6 Algae None None None\n", - "6 7 Olives - processing None None None\n", - "7 8 Corn - all None None None\n", - "8 9 Hay - alfalfa None None None\n", - "9 10 Silage - wheat None None None\n", - "10 11 Rice None None None\n", - "11 12 Peaches None None None\n", - "12 13 Potatoes None None None\n", - "13 14 Cucumbers and pickles None None None\n", - "14 15 Pistachios None None None\n", - "15 16 Cotton None None None\n", - "16 17 Olives - market None None None\n", - "17 18 None None None" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sqlmodel import Session, select, create_engine\n", - "import pandas as pd\n", - "import os\n", - "import sys\n", - "\n", - "\n", - "# Database Connection\n", - "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", - "engine = create_engine(DATABASE_URL)\n", - "print(f\"Connected to database.\")\n", - "\n", - "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", - "\n", - "#reorders columns so id and name are first\n", - "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", - "\n", - "primary_ag_product = primary_ag_product[[*cols]]\n", - "\n", - "primary_ag_product\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "description", - "rawType": "object", - "type": "unknown" - }, - { - "name": "id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "name", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "unknown" - }, - { - "name": "uri", - "rawType": "object", - "type": "unknown" - } - ], - "ref": "52090a35-9379-4dc9-b918-583b23513ddd", - "rows": [ - [ - "0", - null, - "1", - "Tomatoes for processing", - null, - null - ], - [ - "1", - null, - "2", - "Grapes", - null, - null - ], - [ - "2", - null, - "3", - "Almonds", - null, - null - ], - [ - "3", - null, - "4", - "Walnuts", - null, - null - ], - [ - "4", - null, - "5", - "Sweet potatoes", - null, - null - ] - ], - "shape": { - "columns": 5, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
descriptionidnamenoteuri
0None1Tomatoes for processingNoneNone
1None2GrapesNoneNone
2None3AlmondsNoneNone
3None4WalnutsNoneNone
4None5Sweet potatoesNoneNone
\n", - "
" - ], - "text/plain": [ - " description id name note uri\n", - "0 None 1 Tomatoes for processing None None\n", - "1 None 2 Grapes None None\n", - "2 None 3 Almonds None None\n", - "3 None 4 Walnuts None None\n", - "4 None 5 Sweet potatoes None None" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sqlalchemy.orm import Session\n", - "from sqlalchemy import select\n", - "import pandas as pd\n", - "import os\n", - "import sys\n", - "\n", - "# --- project root discovery (unchanged) ---\n", - "path = os.getcwd()\n", - "project_root = None\n", - "while path != os.path.dirname(path):\n", - " if 'pixi.toml' in os.listdir(path):\n", - " project_root = path\n", - " break\n", - " path = os.path.dirname(path)\n", - "\n", - "if not project_root:\n", - " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", - "\n", - "if project_root not in sys.path:\n", - " sys.path.insert(0, project_root)\n", - "\n", - "# --- imports ---\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", - "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", - "\n", - "# --- query + dataframe ---\n", - "with Session(engine) as db:\n", - " stmt = select(*PrimaryAgProduct.__table__.columns)\n", - " rows = db.execute(stmt).mappings().all()\n", - "\n", - "df = pd.DataFrame(rows)\n", - "\n", - "df.head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
21:07:54.222 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "21:07:54.222 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
21:07:59.598 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "21:07:59.598 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
21:07:59.601 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "21:07:59.601 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from sqlalchemy.orm import Session\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", - "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", - " replace_name_with_id_df,\n", - ") \n", - "\n", - "#This extractst the raw proximate data\n", - "df = cmpana.extract(project_root=project_root)\n", - "\n", - "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", - "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", - "\n", - "#this replaces the names with IDs\n", - "with Session(engine) as db:\n", - " parameter_ids = replace_name_with_id_df(\n", - " db=db,\n", - " df=test_df,\n", - " ref_model=Parameter,\n", - " name_column_name=\"name\", # column in df + table\n", - " id_column_name=\"id\", # PK column in table\n", - " final_column_name=\"parameter_id\"\n", - " )\n", - "\n", - "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", - "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", - "\n", - "#resource_id_mapping\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "cmp_uuid_033", - "rawType": "string", - "type": "string" - }, - { - "name": "record_id", - "rawType": "string", - "type": "string" - }, - { - "name": "prepared_sample", - "rawType": "string", - "type": "string" - }, - { - "name": "resource", - "rawType": "string", - "type": "string" - }, - { - "name": "preparation_method", - "rawType": "string", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "string", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "string", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "Int32", - "type": "integer" - }, - { - "name": "repl_id", - "rawType": "string", - "type": "string" - }, - { - "name": "value", - "rawType": "Float32", - "type": "float" - }, - { - "name": "unit", - "rawType": "string", - "type": "string" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "string", - "type": "string" - }, - { - "name": "note", - "rawType": "string", - "type": "string" - }, - { - "name": "analysis_type", - "rawType": "string", - "type": "string" - }, - { - "name": "equipment", - "rawType": "string", - "type": "string" - }, - { - "name": "raw_data_url", - "rawType": "string", - "type": "string" - }, - { - "name": "analyst_email", - "rawType": "string", - "type": "string" - }, - { - "name": "upload_status", - "rawType": "string", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "91fad99c-9137-48ce-af0f-4004b3241e88", - "rows": [ - [ - "0", - "3EE2993D-86E3-1F16-C7EA-F8D555E114", - "(85)E114", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "14.16", - "% dry weight", - "2025-01-23 09:00:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "1", - "46878EF9-1226-22A0-D5D8-CF65E241CB", - "(85)41CB", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "14.18", - "% dry weight", - "2025-01-23 09:00:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "2", - "76A7A2F4-C4E4-E60F-1187-DEC6E02246", - "(85)2246", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "14.12", - "% dry weight", - "2025-01-23 09:00:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "3", - "7A136832-286B-07CB-62DE-ACF52F9311", - "(85)9311", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "15.74", - "% dry weight", - "2025-01-23 09:00:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "4", - "B709ECEE-F9A6-A55D-A59E-93B7B863D7", - "(85)63D7", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "15.75", - "% dry weight", - "2025-01-23 09:01:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "5", - "35080403-BB4B-776B-E045-4627C307BD", - "(85)07BD", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "15.68", - "% dry weight", - "2025-01-23 09:01:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "6", - "D83F3328-931D-582E-B563-4DFF118C05", - "(85)8C05", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "9.88", - "% dry weight", - "2025-01-23 09:01:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "7", - "B3F6AA18-E51A-2585-8417-113DE0F026", - "(85)F026", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "9.25", - "% dry weight", - "2025-01-23 09:01:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "8", - "094CFB14-0D19-C54E-51C4-6BA548A6D4", - "(85)A6D4", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "9.43", - "% dry weight", - "2025-01-23 09:02:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "9", - "99D17377-62B8-2124-5391-62BC1F4630", - "(85)4630", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "11.22", - "% dry weight", - "2025-01-23 09:02:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "10", - "9B784971-B92C-A870-DA69-C4D771B716", - "(85)B716", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "10.51", - "% dry weight", - "2025-01-23 09:02:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "11", - "57C508A2-7489-2AFB-23B2-1630C38BD5", - "(85)8BD5", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "10.72", - "% dry weight", - "2025-01-23 09:02:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "12", - "40EAE6DB-4CD6-3574-949F-70B03A51C8", - "(85)51C8", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "39.54", - "% dry weight", - "2025-01-23 09:03:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "13", - "A752F059-ECF4-FA5B-A9D2-7DF8963329", - "(85)3329", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "39.9", - "% dry weight", - "2025-01-23 09:03:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "14", - "BEB4369F-D40F-DF6A-30F8-75B91C5B1B", - "(85)5B1B", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "40.88", - "% dry weight", - "2025-01-23 09:03:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "15", - "00CAC475-8E18-2318-AA6A-2AE068CADF", - "(D2)CADF", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "19.04", - "% dry weight", - "2025-01-23 09:03:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "16", - "2DA01B52-D301-53C9-220D-A51216BDB6", - "(D2)BDB6", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "18.9", - "% dry weight", - "2025-01-23 09:04:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "17", - "F8D1C324-F3F5-7754-494E-7BBFB262D0", - "(D2)62D0", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "18.98", - "% dry weight", - "2025-01-23 09:04:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "18", - "84FE8C5C-5D37-B0EA-6488-B883CEA7F3", - "(D2)A7F3", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "21.15", - "% dry weight", - "2025-01-23 09:04:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "19", - "E705EAF1-69EB-ADB3-620B-1DD93F4D07", - "(D2)4D07", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "21.0", - "% dry weight", - "2025-01-23 09:04:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "20", - "F6A97D04-E898-06F1-A90B-0EE254D758", - "(D2)D758", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "21.08", - "% dry weight", - "2025-01-23 09:05:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "21", - "6948E00A-4858-6F19-23BC-AA016F12CB", - "(D2)12CB", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "8.84", - "% dry weight", - "2025-01-23 09:05:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "22", - "EEC9EA54-3C1A-6154-B953-401C89CEA7", - "(D2)CEA7", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "8.46", - "% dry weight", - "2025-01-23 09:05:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "23", - "62E4A908-ED27-590F-0DD2-EA254F1E4C", - "(D2)1E4C", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "8.72", - "% dry weight", - "2025-01-23 09:05:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "24", - "AA2BBE0D-FD98-98F9-9880-CDB56142A2", - "(D2)42A2", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "10.04", - "% dry weight", - "2025-01-23 09:06:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "25", - "5186F84F-8BBB-98A7-8FF9-CF85CCD344", - "(D2)D344", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "9.62", - "% dry weight", - "2025-01-23 09:06:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "26", - "2BAF4360-89A7-C605-2141-827901DA56", - "(D2)DA56", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "9.91", - "% dry weight", - "2025-01-23 09:06:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "27", - "B9D049D7-363C-365F-3750-331CEC2FAE", - "(D2)2FAE", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "30.53", - "% dry weight", - "2025-01-23 09:06:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "28", - "4E056928-76F1-99E1-4F78-FD31CC0B71", - "(D2)0B71", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "31.57", - "% dry weight", - "2025-01-23 09:07:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "29", - "4734CD60-9A40-8ECD-7E0D-FA2AD6D96D", - "(D2)D96D", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "31.95", - "% dry weight", - "2025-01-23 09:07:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "30", - "2CD57693-9038-EC9E-EC61-E91793A602", - "(28)A602", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "11.71", - "% dry weight", - "2025-01-23 09:07:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "31", - "BD14399E-2A3B-56A9-AC9C-1689DA75A7", - "(28)75A7", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "13.66", - "% dry weight", - "2025-01-23 09:07:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "32", - "6635AA49-3247-3A08-D00F-CA5447491E", - "(28)491E", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "13.84", - "% dry weight", - "2025-01-23 09:08:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "33", - "EE7D6738-BEE5-87A1-6387-89AE03CFC4", - "(28)CFC4", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "13.01", - "% dry weight", - "2025-01-23 09:08:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "34", - "ACD2BFD6-EA7C-8CBB-F4DA-CAA860F6C9", - "(28)F6C9", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "15.17", - "% dry weight", - "2025-01-23 09:08:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "35", - "1BFB37F6-A5D7-E88E-84A4-DBA38059B5", - "(28)59B5", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "15.37", - "% dry weight", - "2025-01-23 09:08:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "36", - "D6CD55E5-19B7-5E3A-EEE6-95564F8C33", - "(28)8C33", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "9.36", - "% dry weight", - "2025-01-23 09:09:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "37", - "17DE0AED-2CC5-2B5A-E521-049F78F0FE", - "(28)F0FE", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "11.26", - "% dry weight", - "2025-01-23 09:09:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "38", - "4D593E32-1FDA-373B-EE45-38FB01847B", - "(28)847B", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "11.06", - "% dry weight", - "2025-01-23 09:09:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], + "ref": "b4e5431c-6847-48e5-894f-3a2387854a2b", + "rows": [ [ - "39", - "84904964-58B9-A45E-3F79-6747BF7919", - "(28)7919", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "0", + "3ee2993d-86e3-1f16-c7ea-f8d555e114", + "(85)e114", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "1", - "Cmp04xk(28)1", - "10.64", - "% dry weight", - "2025-01-23 09:09:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "40", - "808F6FAC-F9A6-A9EB-56D5-EECA3BA3D2", - "(28)A3D2", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "12.8", - "% dry weight", - "2025-01-23 09:10:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "41", - "67FD9F7C-8CE8-04F4-2CB5-6C42532429", - "(28)2429", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "12.56", + "cmp04xk(85)1", + "glucan", + "14.15999984741211", "% dry weight", - "2025-01-23 09:10:16", + "2025-01-23 09:00:01", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "5" + "ready" ], [ - "42", - "90E8067F-045F-3E6E-5BAA-94BEE9D0CF", - "(28)D0CF", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", "1", - "Cmp04xk(28)1", - "40.12", - "% dry weight", - "2025-01-23 09:10:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "43", - "112546A5-714C-B29F-808B-CBDE6DE48E", - "(28)E48E", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "46878ef9-1226-22a0-d5d8-cf65e241cb", + "(85)41cb", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "2", - "Cmp04xk(28)2", - "39.76", - "% dry weight", - "2025-01-23 09:10:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "44", - "2FF25721-ED5B-05E4-2DC5-5799261266", - "(28)1266", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "40.34", - "% dry weight", - "2025-01-23 09:11:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "45", - "7BC7134E-F8BB-F153-DC50-67A1161509", - "(AB)1509", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(AB)1", - "7.95", + "cmp04xk(85)2", + "glucan", + "14.180000305175781", "% dry weight", - "2025-01-22 09:00:01", + "2025-01-23 09:00:16", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "6" + "ready" ], [ - "46", - "F442FF50-46CF-317D-9708-14F6461273", - "(AB)1273", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", "2", - "Cmp04xk(AB)2", - "7.77", - "% dry weight", - "2025-01-22 09:00:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "47", - "C80175E1-8D21-920C-B982-A246941FCB", - "(AB)1FCB", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "76a7a2f4-c4e4-e60f-1187-dec6e02246", + "(85)2246", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "3", - "Cmp04xk(AB)3", - "7.96", + "cmp04xk(85)3", + "glucan", + "14.119999885559082", "% dry weight", - "2025-01-22 09:00:31", + "2025-01-23 09:00:31", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "6" + "ready" ], [ - "48", - "63CF426B-99C5-3896-C472-49BBAE19CD", - "(AB)19CD", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "3", + "7a136832-286b-07cb-62de-acf52f9311", + "(85)9311", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "1", - "Cmp04xk(AB)1", - "8.84", + "cmp04xk(85)1", + "glucose", + "15.739999771118164", "% dry weight", - "2025-01-22 09:00:46", + "2025-01-23 09:00:46", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "4" + "ready" ], [ - "49", - "A9D902B0-033D-29B4-C477-1A7D3E57F4", - "(AB)57F4", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "4", + "b709ecee-f9a6-a55d-a59e-93b7b863d7", + "(85)63d7", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "2", - "Cmp04xk(AB)2", - "8.64", + "cmp04xk(85)2", + "glucose", + "15.75", "% dry weight", - "2025-01-22 09:01:01", + "2025-01-23 09:01:01", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "4" + "ready" ] ], "shape": { "columns": 21, - "rows": 390 + "rows": 5 } }, "text/html": [ @@ -3772,8 +885,9 @@ " exper_abbrev\n", " repl_no\n", " repl_id\n", - " value\n", + " parameter\n", " ...\n", + " unit\n", " created_at\n", " updated_at\n", " qc_result\n", @@ -3783,353 +897,513 @@ " raw_data_url\n", " analyst_email\n", " upload_status\n", - " parameter_id\n", " \n", " \n", " \n", " \n", " 0\n", - " 3EE2993D-86E3-1F16-C7EA-F8D555E114\n", - " (85)E114\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " 3ee2993d-86e3-1f16-c7ea-f8d555e114\n", + " (85)e114\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 1\n", - " Cmp04xk(85)1\n", - " 14.16\n", + " cmp04xk(85)1\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:01\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 1\n", - " 46878EF9-1226-22A0-D5D8-CF65E241CB\n", - " (85)41CB\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " 46878ef9-1226-22a0-d5d8-cf65e241cb\n", + " (85)41cb\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 2\n", - " Cmp04xk(85)2\n", - " 14.18\n", + " cmp04xk(85)2\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:16\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 2\n", - " 76A7A2F4-C4E4-E60F-1187-DEC6E02246\n", + " 76a7a2f4-c4e4-e60f-1187-dec6e02246\n", " (85)2246\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 3\n", - " Cmp04xk(85)3\n", - " 14.12\n", + " cmp04xk(85)3\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:31\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 3\n", - " 7A136832-286B-07CB-62DE-ACF52F9311\n", + " 7a136832-286b-07cb-62de-acf52f9311\n", " (85)9311\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 1\n", - " Cmp04xk(85)1\n", - " 15.74\n", + " cmp04xk(85)1\n", + " glucose\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:46\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 4\n", " \n", " \n", " 4\n", - " B709ECEE-F9A6-A55D-A59E-93B7B863D7\n", - " (85)63D7\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " b709ecee-f9a6-a55d-a59e-93b7b863d7\n", + " (85)63d7\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 2\n", - " Cmp04xk(85)2\n", - " 15.75\n", + " cmp04xk(85)2\n", + " glucose\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:01:01\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 4\n", " \n", + " \n", + "\n", + "

5 rows × 21 columns

\n", + "" + ], + "text/plain": [ + " cmp_uuid_033 record_id prepared_sample \\\n", + "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 oak-tmpm01o(85) \n", + "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb oak-tmpm01o(85) \n", + "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 oak-tmpm01o(85) \n", + "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 oak-tmpm01o(85) \n", + "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 oak-tmpm01o(85) \n", + "\n", + " resource preparation_method storage_cond exper_abbrev repl_no \\\n", + "0 tomato pomace oven dry rt vacuum sealed cmp04xk 1 \n", + "1 tomato pomace oven dry rt vacuum sealed cmp04xk 2 \n", + "2 tomato pomace oven dry rt vacuum sealed cmp04xk 3 \n", + "3 tomato pomace oven dry rt vacuum sealed cmp04xk 1 \n", + "4 tomato pomace oven dry rt vacuum sealed cmp04xk 2 \n", + "\n", + " repl_id parameter ... unit created_at updated_at \\\n", + "0 cmp04xk(85)1 glucan ... % dry weight 2025-01-23 09:00:01 NaT \n", + "1 cmp04xk(85)2 glucan ... % dry weight 2025-01-23 09:00:16 NaT \n", + "2 cmp04xk(85)3 glucan ... % dry weight 2025-01-23 09:00:31 NaT \n", + "3 cmp04xk(85)1 glucose ... % dry weight 2025-01-23 09:00:46 NaT \n", + "4 cmp04xk(85)2 glucose ... % dry weight 2025-01-23 09:01:01 NaT \n", + "\n", + " qc_result note analysis_type equipment raw_data_url analyst_email \\\n", + "0 pass chemical composition xkang2@lbl.gov \n", + "1 pass chemical composition xkang2@lbl.gov \n", + "2 pass chemical composition xkang2@lbl.gov \n", + "3 pass chemical composition xkang2@lbl.gov \n", + "4 pass chemical composition xkang2@lbl.gov \n", + "\n", + " upload_status \n", + "0 ready \n", + "1 ready \n", + "2 ready \n", + "3 ready \n", + "4 ready \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataframes = [df, df2, df3]\n", + "\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", + "\n", + "clean_dataframes[2].head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "bed16ed0-25aa-43f6-b88b-bace074090d9", + "rows": [ + [ + "0", + null, + "1", + "Tomatoes for processing", + null, + null + ], + [ + "1", + null, + "2", + "Grapes", + null, + null + ], + [ + "2", + null, + "3", + "Almonds", + null, + null + ], + [ + "3", + null, + "4", + "Walnuts", + null, + null + ], + [ + "4", + null, + "5", + "Sweet potatoes", + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
descriptionidnamenoteuri
..................................................................0None1Tomatoes for processingNoneNone
38579799601-0E25-3832-3BBF-F4BD9EE973(A0)E973Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1None2Cmp17xk(A0)217.290001...2025-03-04 09:55:00NaTpassChemical compositionxkang2@lbl.govnot ready5
386A4E3393F-72C3-A0ED-C041-105A33031A(A0)031AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)317.5...2025-03-04 09:55:15NaTpassChemical compositionxkang2@lbl.govnot ready5GrapesNoneNone
3873D5A85F8-31A2-0CC2-74E6-05D8197C3A(A0)7C3AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1Cmp17xk(A0)135.970001...2025-03-04 09:55:30NaTpassChemical compositionxkang2@lbl.govnot ready2None3AlmondsNoneNone
388C7AE456E-4DCD-AE34-A0D0-BD39D04E42(A0)4E42Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)235.279999...2025-03-04 09:55:45NaTpassChemical compositionxkang2@lbl.govnot ready33None4WalnutsNoneNone
389A5E617A2-9831-3E9F-7072-6C5FB8129B(A0)129BHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)334.990002...2025-03-04 09:56:00NaTpassChemical compositionxkang2@lbl.govnot ready34None5Sweet potatoesNoneNone
\n", - "

390 rows × 21 columns

\n", "
" ], "text/plain": [ - " cmp_uuid_033 record_id prepared_sample \\\n", - "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", - "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", - "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", - "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", - "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", - ".. ... ... ... \n", - "385 79799601-0E25-3832-3BBF-F4BD9EE973 (A0)E973 Hum-AlmBr024KM2(A0) \n", - "386 A4E3393F-72C3-A0ED-C041-105A33031A (A0)031A Hum-AlmBr024KM2(A0) \n", - "387 3D5A85F8-31A2-0CC2-74E6-05D8197C3A (A0)7C3A Hum-AlmBr024KM2(A0) \n", - "388 C7AE456E-4DCD-AE34-A0D0-BD39D04E42 (A0)4E42 Hum-AlmBr024KM2(A0) \n", - "389 A5E617A2-9831-3E9F-7072-6C5FB8129B (A0)129B Hum-AlmBr024KM2(A0) \n", - "\n", - " resource preparation_method storage_cond exper_abbrev \\\n", - "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - ".. ... ... ... ... \n", - "385 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "386 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "387 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "388 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "389 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "\n", - " repl_no repl_id value ... created_at updated_at \\\n", - "0 1 Cmp04xk(85)1 14.16 ... 2025-01-23 09:00:01 NaT \n", - "1 2 Cmp04xk(85)2 14.18 ... 2025-01-23 09:00:16 NaT \n", - "2 3 Cmp04xk(85)3 14.12 ... 2025-01-23 09:00:31 NaT \n", - "3 1 Cmp04xk(85)1 15.74 ... 2025-01-23 09:00:46 NaT \n", - "4 2 Cmp04xk(85)2 15.75 ... 2025-01-23 09:01:01 NaT \n", - ".. ... ... ... ... ... ... \n", - "385 2 Cmp17xk(A0)2 17.290001 ... 2025-03-04 09:55:00 NaT \n", - "386 3 Cmp17xk(A0)3 17.5 ... 2025-03-04 09:55:15 NaT \n", - "387 1 Cmp17xk(A0)1 35.970001 ... 2025-03-04 09:55:30 NaT \n", - "388 2 Cmp17xk(A0)2 35.279999 ... 2025-03-04 09:55:45 NaT \n", - "389 3 Cmp17xk(A0)3 34.990002 ... 2025-03-04 09:56:00 NaT \n", - "\n", - " qc_result note analysis_type equipment raw_data_url \\\n", - "0 pass Chemical composition \n", - "1 pass Chemical composition \n", - "2 pass Chemical composition \n", - "3 pass Chemical composition \n", - "4 pass Chemical composition \n", - ".. ... ... ... ... ... \n", - "385 pass Chemical composition \n", - "386 pass Chemical composition \n", - "387 pass Chemical composition \n", - "388 pass Chemical composition \n", - "389 pass Chemical composition \n", - "\n", - " analyst_email upload_status parameter_id \n", - "0 xkang2@lbl.gov ready 6 \n", - "1 xkang2@lbl.gov ready 6 \n", - "2 xkang2@lbl.gov ready 6 \n", - "3 xkang2@lbl.gov ready 4 \n", - "4 xkang2@lbl.gov ready 4 \n", - ".. ... ... ... \n", - "385 xkang2@lbl.gov not ready 5 \n", - "386 xkang2@lbl.gov not ready 5 \n", - "387 xkang2@lbl.gov not ready 3 \n", - "388 xkang2@lbl.gov not ready 3 \n", - "389 xkang2@lbl.gov not ready 3 \n", - "\n", - "[390 rows x 21 columns]" + " description id name note uri\n", + "0 None 1 Tomatoes for processing None None\n", + "1 None 2 Grapes None None\n", + "2 None 3 Almonds None None\n", + "3 None 4 Walnuts None None\n", + "4 None 5 Sweet potatoes None None" ] }, - "execution_count": 15, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "# --- project root discovery (unchanged) ---\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# --- imports ---\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", + "\n", + "# --- query + dataframe ---\n", + "with Session(engine) as db:\n", + " stmt = select(*PrimaryAgProduct.__table__.columns)\n", + " rows = db.execute(stmt).mappings().all()\n", + "\n", + "df = pd.DataFrame(rows)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats = clean_dataframes[0].\\\n", + " groupby(['resource', 'parameter'])['value'].\\\n", + " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", + "\n", + "summary_stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", + " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", + " agg({'value': 'mean'}).\\\n", + " query('value > 30').\\\n", + " sort_values(by='value', ascending=False).\\\n", + " round({'value': 1})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", + "\n", + "def is_it_volatile_solids(df):\n", + " df['check'] = \"VS\"\n", + "\n", + " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", + " return df\n", + "\n", + "is_it_volatile_solids(df)\n", + "\n", + "df[['check', 'parameter']]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", + "def sqrtvalue(df):\n", + " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", + " return df\n", + "\n", + "#List comprehension to apply sqrtvalue to each DataFrame\n", + "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", + "\n", + "# Display the head of the third DataFrame\n", + "clean_rooted_df[2].head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cmpana_raw = cmpana.extract(project_root=project_root)\n", + "\n", + "cmpana_raw.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.orm import Session\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", + " replace_name_with_id_df,\n", + ") \n", + "\n", + "#This extractst the raw proximate data\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", + "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", + "\n", + "#this replaces the names with IDs\n", + "with Session(engine) as db:\n", + " parameter_ids = replace_name_with_id_df(\n", + " db=db,\n", + " df=test_df,\n", + " ref_model=Parameter,\n", + " name_column_name=\"name\", # column in df + table\n", + " id_column_name=\"id\", # PK column in table\n", + " final_column_name=\"parameter_id\"\n", + " )\n", + "\n", + "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", + "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", + "\n", + "#resource_id_mapping\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "parameter_ids\n" ] @@ -4187,6 +1461,63 @@ "\n", "field_sample" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] } ], "metadata": { diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py index d9290968..5d1d22b7 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -40,10 +40,13 @@ def replace_name_with_id_df( model_name_attr: str, id_column_name: str, final_column_name: str, -) -> pd.DataFrame: +) -> tuple[pd.DataFrame, int]: """ Replace a DataFrame name column with foreign key IDs from a SQLAlchemy table. Creates missing reference records if needed. + + Returns: + A tuple containing the modified DataFrame and the number of new records created. """ # 1. Fetch existing reference rows (name + id only) @@ -61,6 +64,7 @@ def replace_name_with_id_df( # 2. Determine which names are new unique_names = set(df_copy[df_name_column].dropna().unique()) new_names = unique_names - set(name_to_id_map.keys()) + num_new_records = len(new_names) # 3. Insert missing reference rows if new_names: @@ -87,4 +91,4 @@ def replace_name_with_id_df( df_copy[final_column_name] = df_copy[df_name_column].map(name_to_id_map) df_copy = df_copy.drop(columns=[df_name_column]) - return df_copy + return df_copy, num_new_records From f4b06ecae3b7e5f59fdb81f9bd288db4ede249f8 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 6 Jan 2026 13:54:03 -0700 Subject: [PATCH 7/7] modified etl_notebook. Db is now running on localhost --- .../pipeline/utils/database_interaction.ipynb | 2 +- .../pipeline/utils/dev_test.ipynb | 53 ++ .../pipeline/utils/etl_notebook.ipynb | 462 +++++++++--------- 3 files changed, 291 insertions(+), 226 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb index 671f288a..c6ba4c99 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb @@ -347,7 +347,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.9" + "version": "3.13.11" } }, "nbformat": 4, diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb new file mode 100644 index 00000000..c7dfc663 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d5df752a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ca_biositing.datamodels.schemas.generated.ca_biositing.Resource" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "import os\n", + "import sys\n", + "from sqlalchemy.orm import Session\n", + "\n", + "from ca_biositing.pipeline.utils.lookup_utils import replace_id_with_name_df\n", + "from ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb index 18fb0314..140d8709 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -20,15 +20,15 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-04 08:53:57,422 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", - "2026-01-04 08:53:57,423 - INFO - Successfully imported all project modules.\n" + "2026-01-06 10:31:31,052 - INFO - Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", + "2026-01-06 10:31:32,479 - INFO - Successfully imported all project modules.\n" ] } ], @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -128,12 +128,14 @@ " if 'updated_at' in df_cleaned.columns:\n", " df_cleaned['updated_at'] = pd.to_datetime(df_cleaned['updated_at'], errors='coerce')\n", "\n", - " # 4. Convert other dtypes to best possible\n", + " # 4. Replace empty strings with NaN so they are properly ignored\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "\n", + " # 5. Convert other dtypes to best possible\n", " df_cleaned = df_cleaned.convert_dtypes()\n", " logger.info('Successfully cleaned DataFrame.')\n", - " \n", - " \n", - " # 5. Convert all string data to lowercase\n", + "\n", + " # 6. Convert all string data to lowercase\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", " logger.info('Converted all string data to lowercase.')\n", " return df_cleaned\n", @@ -154,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -192,7 +194,7 @@ " \n", " try:\n", " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", - " df_normalized = replace_name_with_id_df(\n", + " df_normalized, num_created = replace_name_with_id_df(\n", " db=db,\n", " df=df_normalized,\n", " ref_model=model,\n", @@ -201,6 +203,8 @@ " id_column_name='id',\n", " final_column_name=f'{df_col}_id'\n", " )\n", + " if num_created > 0:\n", + " logger.info(f\"Created {num_created} new records in '{model.__name__}' table.\")\n", " new_col_name = f'{df_col}_id'\n", " num_nulls = df_normalized[new_col_name].isnull().sum()\n", " logger.info(f\"Successfully normalized '{df_col}'. New column '{new_col_name}' contains {num_nulls} null values.\")\n", @@ -219,7 +223,7 @@ " db.rollback()\n", " logger.info('Database session rolled back.')\n", " \n", - " return normalized_dfs" + " return normalized_dfs\n" ] }, { @@ -231,104 +235,112 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-04 08:58:52,849 - INFO - Starting data extraction...\n", - "2026-01-04 08:58:52,865 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:52,877 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:54,699 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:54,703 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:54,724 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:54,736 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:55,992 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:55,996 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:56,016 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:56,029 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:57,291 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:57,294 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:57,296 - INFO - Data extraction complete.\n", - "2026-01-04 08:58:57,299 - INFO - Starting data cleaning...\n", - "2026-01-04 08:58:57,299 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,302 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,309 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:40,834 - INFO - Starting data extraction...\n", + "2026-01-06 10:31:40,881 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:40,903 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:42,948 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/csrf-token?client=ae9812ec-079b-4fe5-845c-493121391421 \"HTTP/1.1 422 Unprocessable Entity\"\n", + "2026-01-06 10:31:42,966 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:45,659 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:45,661 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:45,672 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:45,680 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:46,995 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:47,292 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:47,296 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:47,313 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:47,326 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:48,924 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:48,929 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:48,930 - INFO - Data extraction complete.\n", + "2026-01-06 10:31:48,931 - INFO - Starting data cleaning...\n", + "2026-01-06 10:31:48,933 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,939 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,950 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,314 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,315 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,316 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,319 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:48,958 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,958 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,959 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,962 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,320 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,321 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,325 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,335 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:48,964 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,964 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,965 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,969 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,340 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,342 - INFO - Data cleaning complete.\n", - "2026-01-04 08:58:57,342 - INFO - Starting data normalization...\n", - "2026-01-04 08:58:57,343 - INFO - Starting normalization process for 3 dataframes.\n", - "2026-01-04 08:58:57,343 - INFO - Processing DataFrame #1 with 759 rows.\n", - "2026-01-04 08:58:57,344 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,355 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,356 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,367 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,367 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,369 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,369 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,372 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,372 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,374 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,374 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,375 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,375 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,379 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,379 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,379 - INFO - Finished processing DataFrame #1.\n", - "2026-01-04 08:58:57,379 - INFO - Processing DataFrame #2 with 64 rows.\n", - "2026-01-04 08:58:57,380 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,384 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,384 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,388 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,388 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,392 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,392 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,396 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,396 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,398 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,398 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,399 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,400 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,403 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,403 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,403 - INFO - Finished processing DataFrame #2.\n", - "2026-01-04 08:58:57,403 - INFO - Processing DataFrame #3 with 390 rows.\n", - "2026-01-04 08:58:57,404 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,406 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,406 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,408 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,408 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,409 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,410 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,411 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,411 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,413 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,413 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,415 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,415 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,416 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,416 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,417 - INFO - Finished processing DataFrame #3.\n", - "2026-01-04 08:58:57,417 - INFO - Committing database session.\n", - "2026-01-04 08:58:57,418 - INFO - Database commit successful.\n", - "2026-01-04 08:58:57,418 - INFO - Data normalization complete.\n", - "2026-01-04 08:58:57,418 - INFO - Displaying results of normalization...\n" + "2026-01-06 10:31:48,972 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,973 - INFO - Data cleaning complete.\n", + "2026-01-06 10:31:48,973 - INFO - Starting data normalization...\n", + "2026-01-06 10:31:48,974 - INFO - Starting normalization process for 3 dataframes.\n", + "2026-01-06 10:31:48,974 - INFO - Processing DataFrame #1 with 1030 rows.\n", + "2026-01-06 10:31:48,974 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,016 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:49,116 - INFO - Created 35 new records in 'Resource' table.\n", + "2026-01-06 10:31:49,116 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,117 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,120 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,120 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,123 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,124 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,144 - INFO - Created 1 new records in 'Parameter' table.\n", + "2026-01-06 10:31:49,145 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,147 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,167 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,176 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,182 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,182 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,189 - INFO - Created 1 new records in 'AnalysisType' table.\n", + "2026-01-06 10:31:49,189 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,189 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,190 - INFO - Finished processing DataFrame #1.\n", + "2026-01-06 10:31:49,190 - INFO - Processing DataFrame #2 with 64 rows.\n", + "2026-01-06 10:31:49,191 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,197 - INFO - Created 1 new records in 'Resource' table.\n", + "2026-01-06 10:31:49,198 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,198 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,201 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,202 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,204 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,204 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,206 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,206 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,209 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,209 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,211 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,212 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,213 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,213 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,214 - INFO - Finished processing DataFrame #2.\n", + "2026-01-06 10:31:49,214 - INFO - Processing DataFrame #3 with 390 rows.\n", + "2026-01-06 10:31:49,215 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,217 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,217 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,220 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,220 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,222 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,222 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,224 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,224 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,225 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,225 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,228 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,228 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,230 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,230 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,230 - INFO - Finished processing DataFrame #3.\n", + "2026-01-06 10:31:49,230 - INFO - Committing database session.\n", + "2026-01-06 10:31:49,232 - INFO - Database commit successful.\n", + "2026-01-06 10:31:49,241 - INFO - Data normalization complete.\n", + "2026-01-06 10:31:49,244 - INFO - Displaying results of normalization...\n" ] }, { @@ -448,7 +460,7 @@ "type": "integer" } ], - "ref": "e2d8eb1c-3fc8-4b97-99b0-df76b598d9df", + "ref": "b0e10f3c-2f75-4628-acc4-1c383e10ca93", "rows": [ [ "0", @@ -465,13 +477,13 @@ "pass", "not ready", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "1", @@ -488,13 +500,13 @@ "pass", "ready", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "2", @@ -511,13 +523,13 @@ "pass", "imported", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "3", @@ -534,13 +546,13 @@ "pass", "import failed", "", - "24", + "6", "109", "7", - "23", + "4", "1", "1", - "6" + "1" ], [ "4", @@ -557,13 +569,13 @@ "pass", "", "", - "24", + "6", "109", "7", - "23", + "4", "1", "1", - "6" + "1" ] ], "shape": { @@ -628,13 +640,13 @@ " pass\n", " not ready\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 1\n", @@ -651,13 +663,13 @@ " pass\n", " ready\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 2\n", @@ -674,13 +686,13 @@ " pass\n", " imported\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 3\n", @@ -697,13 +709,13 @@ " pass\n", " import failed\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 23\n", + " 4\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 4\n", @@ -720,13 +732,13 @@ " pass\n", " \n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 23\n", + " 4\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", "\n", @@ -748,25 +760,25 @@ "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", "\n", " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", - "0 NaT pass not ready 24 109 \n", - "1 NaT pass ready 24 109 \n", - "2 NaT pass imported 24 109 \n", - "3 NaT pass import failed 24 109 \n", - "4 NaT pass 24 109 \n", + "0 NaT pass not ready 6 109 \n", + "1 NaT pass ready 6 109 \n", + "2 NaT pass imported 6 109 \n", + "3 NaT pass import failed 6 109 \n", + "4 NaT pass 6 109 \n", "\n", " preparation_method_id parameter_id unit_id analyst_email_id \\\n", - "0 7 25 1 1 \n", - "1 7 25 1 1 \n", - "2 7 25 1 1 \n", - "3 7 23 1 1 \n", - "4 7 23 1 1 \n", + "0 7 3 1 1 \n", + "1 7 3 1 1 \n", + "2 7 3 1 1 \n", + "3 7 4 1 1 \n", + "4 7 4 1 1 \n", "\n", " analysis_type_id \n", - "0 6 \n", - "1 6 \n", - "2 6 \n", - "3 6 \n", - "4 6 " + "0 1 \n", + "1 1 \n", + "2 1 \n", + "3 1 \n", + "4 1 " ] }, "metadata": {}, @@ -899,7 +911,7 @@ "type": "integer" } ], - "ref": "323960ff-b723-432d-b7c9-fa0f9dd8c185", + "ref": "0f732e49-22cb-4b72-a52a-9fade9af05ef", "rows": [ [ "0", @@ -918,13 +930,13 @@ "", "", "", - "8", + "27", "80", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "1", @@ -943,13 +955,13 @@ "", "", "", - "8", + "27", "80", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "2", @@ -968,13 +980,13 @@ "", "", "", - "26", + "11", "96", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "3", @@ -993,13 +1005,13 @@ "", "", "", - "2", + "12", "91", "9", - "28", + "8", "3", "2", - "7" + "2" ], [ "4", @@ -1018,13 +1030,13 @@ "", "", "", - "7", + "21", "153", "9", - "28", + "8", "3", "2", - "7" + "2" ] ], "shape": { @@ -1091,13 +1103,13 @@ " \n", " \n", " \n", - " 8\n", + " 27\n", " 80\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 1\n", @@ -1115,13 +1127,13 @@ " \n", " \n", " \n", - " 8\n", + " 27\n", " 80\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 2\n", @@ -1139,13 +1151,13 @@ " \n", " \n", " \n", - " 26\n", + " 11\n", " 96\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 3\n", @@ -1163,13 +1175,13 @@ " \n", " \n", " \n", - " 2\n", + " 12\n", " 91\n", " 9\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 4\n", @@ -1187,13 +1199,13 @@ " \n", " \n", " \n", - " 7\n", + " 21\n", " 153\n", " 9\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", "\n", @@ -1216,25 +1228,25 @@ "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", "\n", " updated_at ... equipment raw_data_url upload_status resource_id \\\n", - "0 NaT ... 8 \n", - "1 NaT ... 8 \n", - "2 NaT ... 26 \n", - "3 NaT ... 2 \n", - "4 NaT ... 7 \n", + "0 NaT ... 27 \n", + "1 NaT ... 27 \n", + "2 NaT ... 11 \n", + "3 NaT ... 12 \n", + "4 NaT ... 21 \n", "\n", " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 80 6 28 3 \n", - "1 80 6 28 3 \n", - "2 96 6 28 3 \n", - "3 91 9 28 3 \n", - "4 153 9 28 3 \n", + "0 80 6 8 3 \n", + "1 80 6 8 3 \n", + "2 96 6 8 3 \n", + "3 91 9 8 3 \n", + "4 153 9 8 3 \n", "\n", " analyst_email_id analysis_type_id \n", - "0 2 7 \n", - "1 2 7 \n", - "2 2 7 \n", - "3 2 7 \n", - "4 2 7 \n", + "0 2 2 \n", + "1 2 2 \n", + "2 2 2 \n", + "3 2 2 \n", + "4 2 2 \n", "\n", "[5 rows x 22 columns]" ] @@ -1364,7 +1376,7 @@ "type": "integer" } ], - "ref": "31e4fb08-8acb-41fc-bdec-4f79aea24d5f", + "ref": "95361c64-4a65-4fd6-bde6-2efae692d84c", "rows": [ [ "0", @@ -1382,10 +1394,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1406,10 +1418,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1430,10 +1442,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1454,10 +1466,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "14", + "11", "2", "1", "3" @@ -1478,10 +1490,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "14", + "11", "2", "1", "3" @@ -1551,10 +1563,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1575,10 +1587,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1599,10 +1611,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1623,10 +1635,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 14\n", + " 11\n", " 2\n", " 1\n", " 3\n", @@ -1647,10 +1659,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 14\n", + " 11\n", " 2\n", " 1\n", " 3\n", @@ -1676,18 +1688,18 @@ "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", "\n", " qc_result ... equipment raw_data_url upload_status resource_id \\\n", - "0 pass ... ready 24 \n", - "1 pass ... ready 24 \n", - "2 pass ... ready 24 \n", - "3 pass ... ready 24 \n", - "4 pass ... ready 24 \n", + "0 pass ... ready 6 \n", + "1 pass ... ready 6 \n", + "2 pass ... ready 6 \n", + "3 pass ... ready 6 \n", + "4 pass ... ready 6 \n", "\n", " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 92 8 12 2 \n", - "1 92 8 12 2 \n", - "2 92 8 12 2 \n", - "3 92 8 14 2 \n", - "4 92 8 14 2 \n", + "0 92 8 13 2 \n", + "1 92 8 13 2 \n", + "2 92 8 13 2 \n", + "3 92 8 11 2 \n", + "4 92 8 11 2 \n", "\n", " analyst_email_id analysis_type_id \n", "0 1 3 \n", @@ -1765,9 +1777,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Pixi Default Environment", + "display_name": "default", "language": "python", - "name": "pixi-default" + "name": "python3" }, "language_info": { "codemirror_mode": {