From 5574630951e0d6a65d0bbdadda22308f444d122f Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Mon, 22 Dec 2025 14:55:55 -0700 Subject: [PATCH 01/10] adding comment to init.py to try to solve CI CD issues --- .../datamodels/ca_biositing/datamodels/schemas/__init__.py | 1 + .../ca_biositing/datamodels/schemas/generated/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py index dbb70e85..6279548e 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/__init__.py @@ -1 +1,2 @@ # This file makes this a Python package. +# Schemas package. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py index dbb70e85..73de05be 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/__init__.py @@ -1 +1,2 @@ # This file makes this a Python package. +# Generated schemas. From e0947fc73d0de22736c4a7c117719f7a7df9a42c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 23 Dec 2025 10:12:53 -0700 Subject: [PATCH 02/10] fixed primary_ag_product etl pipeline, modified master flow script and docs --- AGENTS.md | 2 +- ...28_changing_primary_crop_to_primary_ag_.py | 56 +++++++++++++++++++ docs/README.md | 2 +- docs/datamodels/README.md | 2 +- docs/deployment/README.md | 2 +- docs/pipeline/README.md | 2 +- docs/resources/README.md | 2 +- docs/webservice/README.md | 2 +- resources/prefect/run_prefect_flow.py | 30 ++++++---- .../linkml/modules/external_data.yaml | 10 ++-- .../linkml/modules/resource_information.yaml | 10 ++-- .../schemas/generated/aim1_records.py | 12 ++-- .../schemas/generated/aim2_records.py | 12 ++-- .../schemas/generated/ca_biositing.py | 16 +++--- .../generated/data_sources_metadata.py | 12 ++-- .../schemas/generated/experiment_equipment.py | 12 ++-- .../schemas/generated/external_data.py | 16 +++--- .../schemas/generated/field_sampling.py | 12 ++-- .../schemas/generated/general_analysis.py | 12 ++-- .../schemas/generated/infrastructure.py | 12 ++-- .../generated/methods_parameters_units.py | 12 ++-- .../schemas/generated/resource_information.py | 12 ++-- .../schemas/generated/sample_preparation.py | 12 ++-- src/ca_biositing/pipeline/AGENTS.md | 24 +++++--- src/ca_biositing/pipeline/README.md | 35 ++++++------ ...imary_product.py => primary_ag_product.py} | 24 ++++---- .../transform/products/primary_ag_product.py | 41 ++++++++++++++ .../etl/transform/products/primary_product.py | 38 ------------- .../pipeline/flows/primary_ag_product.py | 26 +++++++++ .../pipeline/flows/primary_product.py | 26 --------- 30 files changed, 280 insertions(+), 206 deletions(-) create mode 100644 alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py rename src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/{primary_product.py => primary_ag_product.py} (61%) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py diff --git a/AGENTS.md b/AGENTS.md index 45e1e109..f65c5373 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -404,7 +404,7 @@ ca-biositing/ │ │ │ │ ├── transform/ # Transform tasks │ │ │ │ └── load/ # Load tasks │ │ │ ├── flows/ # Prefect flows -│ │ │ │ ├── primary_product.py +│ │ │ │ ├── primary_ag_product.py │ │ │ │ ├── analysis_type.py │ │ │ │ └── ... │ │ │ └── utils/ # Utilities diff --git a/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py b/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py new file mode 100644 index 00000000..d7efd056 --- /dev/null +++ b/alembic/versions/e84690898528_changing_primary_crop_to_primary_ag_.py @@ -0,0 +1,56 @@ +"""changing primary crop to primary_ag_product + +Revision ID: e84690898528 +Revises: fc422d3d9ea6 +Create Date: 2025-12-23 16:04:40.644736 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e84690898528' +down_revision: Union[str, Sequence[str], None] = 'fc422d3d9ea6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('primary_ag_product', + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.drop_table('primary_crop') + op.add_column('resource', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True)) + op.drop_column('resource', 'primary_crop_id') + op.add_column('resource_usda_commodity_map', sa.Column('primary_ag_product_id', sa.Integer(), nullable=True)) + op.drop_column('resource_usda_commodity_map', 'primary_crop_id') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('resource_usda_commodity_map', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column('resource_usda_commodity_map', 'primary_ag_product_id') + op.add_column('resource', sa.Column('primary_crop_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column('resource', 'primary_ag_product_id') + op.create_table('primary_crop', + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('primary_crop_pkey')) + ) + op.drop_table('primary_ag_product') + # ### end Alembic commands ### diff --git a/docs/README.md b/docs/README.md index 94389aee..32d46ee8 120000 --- a/docs/README.md +++ b/docs/README.md @@ -1 +1 @@ -../README.md +../README.md \ No newline at end of file diff --git a/docs/datamodels/README.md b/docs/datamodels/README.md index f3c7aa23..b62101ad 120000 --- a/docs/datamodels/README.md +++ b/docs/datamodels/README.md @@ -1 +1 @@ -../../src/ca_biositing/datamodels/README.md +../../src/ca_biositing/datamodels/README.md \ No newline at end of file diff --git a/docs/deployment/README.md b/docs/deployment/README.md index 5159bcfe..d5c46ad4 120000 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -1 +1 @@ -../../deployment/README.md +../../deployment/README.md \ No newline at end of file diff --git a/docs/pipeline/README.md b/docs/pipeline/README.md index 544d50b1..82a8c831 120000 --- a/docs/pipeline/README.md +++ b/docs/pipeline/README.md @@ -1 +1 @@ -../../src/ca_biositing/pipeline/README.md +../../src/ca_biositing/pipeline/README.md \ No newline at end of file diff --git a/docs/resources/README.md b/docs/resources/README.md index ea34ca59..9f21ec35 120000 --- a/docs/resources/README.md +++ b/docs/resources/README.md @@ -1 +1 @@ -../../resources/README.md +../../resources/README.md \ No newline at end of file diff --git a/docs/webservice/README.md b/docs/webservice/README.md index 24ea2883..749a635e 120000 --- a/docs/webservice/README.md +++ b/docs/webservice/README.md @@ -1 +1 @@ -../../src/ca_biositing/webservice/README.md +../../src/ca_biositing/webservice/README.md \ No newline at end of file diff --git a/resources/prefect/run_prefect_flow.py b/resources/prefect/run_prefect_flow.py index b350f382..03f5e926 100644 --- a/resources/prefect/run_prefect_flow.py +++ b/resources/prefect/run_prefect_flow.py @@ -1,24 +1,32 @@ import sys -from prefect import flow -from ca_biositing.pipeline.flows.primary_product import primary_product_flow -from ca_biositing.pipeline.flows.analysis_type import analysis_type_flow +import traceback +from prefect import flow, get_run_logger +from prefect.utilities.importtools import import_object -# A dictionary mapping flow names to their function objects +# A dictionary mapping flow names to their import paths AVAILABLE_FLOWS = { - "primary_product": primary_product_flow, - "analysis_type": analysis_type_flow, + "primary_ag_product": "ca_biositing.pipeline.flows.primary_ag_product.primary_ag_product_flow", + "analysis_type": "ca_biositing.pipeline.flows.analysis_type.analysis_type_flow", } @flow(name="Master ETL Flow", log_prints=True) def master_flow(): """ A master flow to orchestrate all ETL pipelines. + This flow dynamically imports and runs sub-flows, allowing it to continue + even if some sub-flows fail to import or run. """ - print("Running master ETL flow...") - for flow_name, flow_func in AVAILABLE_FLOWS.items(): - print(f"--- Running sub-flow: {flow_name} ---") - flow_func() - print("Master ETL flow completed.") + logger = get_run_logger() + logger.info("Running master ETL flow...") + for flow_name, flow_path in AVAILABLE_FLOWS.items(): + try: + logger.info(f"--- Running sub-flow: {flow_name} ---") + flow_func = import_object(flow_path) + flow_func() + except Exception as e: + logger.error(f"Flow '{flow_name}' failed with error: {e}") + logger.error(traceback.format_exc()) + logger.info("Master ETL flow completed.") if __name__ == "__main__": # This script is a placeholder for running flows directly. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml index 3321a2f4..679a5f4a 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml @@ -84,7 +84,7 @@ classes: description: Mapping between resources/crops and USDA commodities. slots: - resource_id - - primary_crop_id + - primary_ag_product_id - usda_commodity_id - match_tier - note @@ -354,19 +354,19 @@ slots: main_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. secondary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. tertiary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. quaternary_crop: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. confidence: range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml index f06fa29d..82237cae 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml @@ -22,7 +22,7 @@ classes: description: Biomass resource definition. slots: - name - - primary_crop_id + - primary_ag_product_id - resource_class_id - resource_subclass_id - note @@ -36,9 +36,9 @@ classes: is_a: LookupBase description: Sub-classification of resources. - PrimaryCrop: + PrimaryAgProduct: is_a: LookupBase - description: Primary crop definition. + description: Primary agricultural product definition. slots: - note @@ -89,9 +89,9 @@ classes: slots: #Resource slots - primary_crop_id: + primary_ag_product_id: range: integer - description: Reference to PrimaryCrop. + description: Reference to PrimaryAgProduct. resource_class_id: range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py index 075f0de2..85b02306 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py @@ -892,7 +892,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -905,7 +905,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -967,11 +967,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -981,7 +981,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py index 45edb52c..374bbf8b 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py @@ -918,7 +918,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -931,7 +931,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -993,11 +993,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1007,7 +1007,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py index 734d1d17..6fc7680f 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py @@ -738,7 +738,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -751,7 +751,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -813,11 +813,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -827,7 +827,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2206,7 +2206,7 @@ class ResourceUsdaCommodityMap(BaseEntity): __tablename__ = 'resource_usda_commodity_map' resource_id = Column(Integer()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) usda_commodity_id = Column(Integer()) match_tier = Column(Text()) note = Column(Text()) @@ -2218,7 +2218,7 @@ class ResourceUsdaCommodityMap(BaseEntity): def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_crop_id={self.primary_crop_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py index 9183431c..10206eab 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py @@ -383,7 +383,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -396,7 +396,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -458,11 +458,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -472,7 +472,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py index ae56a768..d27ecc16 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py @@ -1078,7 +1078,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -1091,7 +1091,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1153,11 +1153,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1167,7 +1167,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py index d854df67..deaaf7e6 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py @@ -358,7 +358,7 @@ class ResourceUsdaCommodityMap(BaseEntity): __tablename__ = 'resource_usda_commodity_map' resource_id = Column(Integer()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) usda_commodity_id = Column(Integer()) match_tier = Column(Text()) note = Column(Text()) @@ -370,7 +370,7 @@ class ResourceUsdaCommodityMap(BaseEntity): def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_crop_id={self.primary_crop_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1041,7 +1041,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -1054,7 +1054,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1116,11 +1116,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1130,7 +1130,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py index 822de073..0e5c3b47 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py @@ -512,7 +512,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -525,7 +525,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -587,11 +587,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -601,7 +601,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py index 6470faba..5e8fd492 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py @@ -501,7 +501,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -514,7 +514,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -576,11 +576,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -590,7 +590,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py index 796558e4..021a9423 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py @@ -860,7 +860,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -873,7 +873,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -935,11 +935,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -949,7 +949,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py index 6be929f8..85ddde97 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py @@ -416,7 +416,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -429,7 +429,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -491,11 +491,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -505,7 +505,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py index bc84a0af..627f9124 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py @@ -99,7 +99,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -112,7 +112,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -174,11 +174,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -188,7 +188,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py index b543fbaa..ae68592c 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py @@ -317,7 +317,7 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_crop_id = Column(Integer()) + primary_ag_product_id = Column(Integer()) resource_class_id = Column(Integer()) resource_subclass_id = Column(Integer()) note = Column(Text()) @@ -330,7 +330,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_crop_id={self.primary_crop_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -392,11 +392,11 @@ def __repr__(self): -class PrimaryCrop(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Primary crop definition. + Primary agricultural product definition. """ - __tablename__ = 'primary_crop' + __tablename__ = 'primary_ag_product' note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -406,7 +406,7 @@ class PrimaryCrop(LookupBase): def __repr__(self): - return f"PrimaryCrop(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/pipeline/AGENTS.md b/src/ca_biositing/pipeline/AGENTS.md index 16b5e29a..1b9fc453 100644 --- a/src/ca_biositing/pipeline/AGENTS.md +++ b/src/ca_biositing/pipeline/AGENTS.md @@ -109,11 +109,13 @@ Located in `ca_biositing/pipeline/etl/`: - Pattern: Prefect `@task` decorated functions that return DataFrames 2. **`transform/`** - Data transformation tasks - - `products/primary_product.py`: Transform primary product data + - `products/primary_ag_product.py`: Transform primary agricultural product + data - Pattern: Prefect `@task` functions that accept and return DataFrames 3. **`load/`** - Data loading tasks - - `products/primary_product.py`: Load primary products to database + - `products/primary_ag_product.py`: Load primary agricultural products to + database - `analysis/`: Analysis-related load functions - Pattern: Prefect `@task` functions that insert data via SQLModel session @@ -126,7 +128,7 @@ Located in `ca_biositing/pipeline/etl/`: Located in `ca_biositing/pipeline/flows/`: -- **`primary_product.py`**: Primary product ETL flow +- **`primary_ag_product.py`**: Primary agricultural product ETL flow - **`analysis_type.py`**: Analysis type ETL flow - Pattern: Prefect `@flow` decorated functions that orchestrate tasks @@ -141,8 +143,10 @@ Located in `ca_biositing/pipeline/utils/`: 2. **`gsheet_to_pandas.py`** - Google Sheets to DataFrame conversion - `gsheet_to_df()`: Main extraction function -3. **`run_pipeline.py`** - Master pipeline orchestrator - - Runs all ETL flows in sequence +3. **`../../resources/prefect/run_prefect_flow.py`** - Master pipeline + orchestrator + - A resilient master flow that dynamically imports and runs all ETL flows, + catching errors from individual flows. 4. **`clear_alembic.py`** - Database migration cleanup utility @@ -442,7 +446,8 @@ def test_lookup_utility(session): 4. **Create flow** in `flows/my_flow.py`: - Use `@flow` decorator - Orchestrate extract → transform → load - - Add to master orchestrator in `utils/run_pipeline.py` + - Add the flow's import path to the `AVAILABLE_FLOWS` dictionary in + `resources/prefect/run_prefect_flow.py` 5. **Write tests** in `tests/test_my_pipeline.py`: - Mock external dependencies @@ -634,10 +639,11 @@ Run flows from command line or Python: ```bash # Run specific flow -python -m ca_biositing.pipeline.flows.primary_product +python -m ca_biositing.pipeline.flows.primary_ag_product -# Run all flows (master orchestrator) -python -m ca_biositing.pipeline.utils.run_pipeline +# Run all flows via the main project's pixi task +# This executes the master flow defined in resources/prefect/run_prefect_flow.py +pixi run run-etl ``` ### Usage with Docker diff --git a/src/ca_biositing/pipeline/README.md b/src/ca_biositing/pipeline/README.md index ac9c59e1..e0514b5f 100644 --- a/src/ca_biositing/pipeline/README.md +++ b/src/ca_biositing/pipeline/README.md @@ -33,20 +33,20 @@ src/ca_biositing/pipeline/ │ │ │ └── experiments.py │ │ ├── transform/ # Data transformation tasks │ │ │ └── products/ -│ │ │ └── primary_product.py +│ │ │ └── primary_ag_product.py │ │ ├── load/ # Data loading tasks │ │ │ ├── analysis/ │ │ │ └── products/ -│ │ │ └── primary_product.py +│ │ │ └── primary_ag_product.py │ │ └── templates/ # ETL module templates │ ├── flows/ # Prefect flow definitions │ │ ├── analysis_type.py -│ │ └── primary_product.py +│ │ └── primary_ag_product.py │ └── utils/ # Utility functions │ ├── __init__.py │ ├── gsheet_to_pandas.py │ ├── lookup_utils.py -│ └── run_pipeline.py +│ └── ../../resources/prefect/run_prefect_flow.py ├── tests/ # Test suite │ ├── __init__.py │ ├── conftest.py # Pytest fixtures @@ -89,15 +89,12 @@ directory: on SQLModel changes - **Note:** Database models are now in the shared `ca-biositing-datamodels` package -- **[See: docs/ALEMBIC_WORKFLOW.md](./ALEMBIC_WORKFLOW.md)** ### 3. ETL Pipeline Development (Prefect) - **Purpose:** Running the ETL pipeline and adding new data pipelines - **Details:** Using Prefect's flow orchestration with extract, transform, and load tasks -- **[See: docs/ETL_WORKFLOW.md](./ETL_WORKFLOW.md)** -- **[See: docs/PREFECT_WORKFLOW.md](./PREFECT_WORKFLOW.md)** ### 4. Google Cloud Setup @@ -162,7 +159,7 @@ See `tests/README.md` for detailed information about the test suite. ```python from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info -from ca_biositing.pipeline.flows.primary_product import primary_product_flow +from ca_biositing.pipeline.flows.primary_ag_product import primary_ag_product_flow from ca_biositing.pipeline.utils.lookup_utils import replace_name_with_id_df # Use in your code @@ -172,10 +169,10 @@ from ca_biositing.pipeline.utils.lookup_utils import replace_name_with_id_df ### Running Prefect Flows ```python -from ca_biositing.pipeline.flows.primary_product import primary_product_flow +from ca_biositing.pipeline.flows.primary_ag_product import primary_ag_product_flow # Run the flow -primary_product_flow() +primary_ag_product_flow() ``` ### Using Utility Functions @@ -232,7 +229,8 @@ docker-compose exec app alembic upgrade head **5. Run ETL Pipeline:** ```bash -docker-compose exec app python utils/run_pipeline.py +# This executes the master flow defined in resources/prefect/run_prefect_flow.py +pixi run run-etl ``` See `docs/DOCKER_WORKFLOW.md` and `docs/ETL_WORKFLOW.md` for detailed @@ -249,17 +247,19 @@ instructions. **Transform:** Data cleaning and transformation using pandas -- `transform/products/primary_product.py`: Transform primary product data +- `transform/products/primary_ag_product.py`: Transform primary agricultural + product data **Load:** Load transformed data into PostgreSQL -- `load/products/primary_product.py`: Load primary products into database +- `load/products/primary_ag_product.py`: Load primary agricultural products into + database ### Prefect Flows Orchestrated workflows that combine ETL tasks: -- `flows/primary_product.py`: Primary product ETL flow +- `flows/primary_ag_product.py`: Primary agricultural product ETL flow - `flows/analysis_type.py`: Analysis type ETL flow ### Utility Functions @@ -271,7 +271,9 @@ Orchestrated workflows that combine ETL tasks: **`gsheet_to_pandas.py`**: Google Sheets to pandas DataFrame conversion -**`run_pipeline.py`**: Master script to run all ETL flows +**`../../resources/prefect/run_prefect_flow.py`**: A resilient master flow that +dynamically imports and runs all ETL flows, catching errors from individual +flows. ## Dependencies @@ -304,8 +306,7 @@ pixi run pre-commit run --files src/ca_biositing/pipeline/**/* 5. **Tests:** Add tests in `tests/` 6. **Run:** Execute the flow -See `etl/templates/` for template files and `docs/ETL_WORKFLOW.md` for detailed -instructions. +See `etl/templates/` for template files ### Adding Database Models diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py similarity index 61% rename from src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py rename to src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py index 57d0d6d3..0508d3a3 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_product.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/load/products/primary_ag_product.py @@ -2,37 +2,37 @@ from prefect import task, get_run_logger from sqlmodel import Session, select from ca_biositing.datamodels.database import engine -from ca_biositing.datamodels.biomass import PrimaryProduct +from ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct @task -def load_products_primary_product(primary_product_df: pd.DataFrame): +def load_products_primary_ag_product(primary_ag_product_df: pd.DataFrame): """ - Loads the data from the primary products DataFrame into the database. + Loads the data from the primary ag products DataFrame into the database. - Iterates over a DataFrame and inserts each product name from the 'Primary_crop' column - into the PrimaryProduct table. + Iterates over a DataFrame and inserts each product name from the 'name' column + into the PrimaryAgProduct table. """ logger = get_run_logger() - if primary_product_df is None or primary_product_df.empty: + if primary_ag_product_df is None or primary_ag_product_df.empty: logger.info("No data to load. Skipping database insertion.") return - column_name = 'Primary_crop' - if column_name not in primary_product_df.columns: + column_name = 'name' + if column_name not in primary_ag_product_df.columns: logger.error(f"Column '{column_name}' not found in the DataFrame. Aborting load.") return - logger.info(f"Attempting to load {len(primary_product_df)} products into the database...") + logger.info(f"Attempting to load {len(primary_ag_product_df)} products into the database...") with Session(engine) as session: - statement = select(PrimaryProduct.primary_product_name) + statement = select(PrimaryAgProduct.name) existing_products = session.exec(statement).all() existing_product_names = set(existing_products) records_to_add = [] - for product_name in primary_product_df[column_name]: + for product_name in primary_ag_product_df[column_name]: if product_name not in existing_product_names: - product = PrimaryProduct(primary_product_name=product_name) + product = PrimaryAgProduct(name=product_name) records_to_add.append(product) existing_product_names.add(product_name) # Add to set to avoid re-adding in same batch diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py new file mode 100644 index 00000000..25182c62 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_ag_product.py @@ -0,0 +1,41 @@ +from typing import Optional, Dict +import pandas as pd +from prefect import task, get_run_logger + +EXTRACT_SOURCES = ["basic_sample_info"] + +@task +def transform_products_primary_ag_product(data_sources: Dict[str, pd.DataFrame]) -> Optional[pd.DataFrame]: + """ + Transforms the raw data to extract unique primary agricultural products. + + This function is purely for transformation (the 'T' in ETL). It takes the + raw DataFrame and performs the specific transformations needed for the + 'primary_ag_product' table. + + Args: + data_sources: A dictionary of DataFrames from the extraction step. + + Returns: + A transformed DataFrame with a single 'name' column containing + unique product names, or None if an error occurs. + """ + logger = get_run_logger() + logger.info("Transforming raw data for primary ag products...") + + raw_df = data_sources["basic_sample_info"] + + #Step 1: Convert column names to lowercase for consistency + raw_df.columns = [col.lower() for col in raw_df.columns] + + # Step 2: Check if the required column exists + if 'primary_ag_product' not in raw_df.columns: + logger.error("'primary_ag_product' column not found in the raw data.") + return None + + # Step 3: Get unique product names and create the final DataFrame + primary_ag_product= raw_df['primary_ag_product'].unique() + transformed_df = pd.DataFrame(primary_ag_product, columns=["name"]) + + logger.info(f"Successfully transformed data, found {len(transformed_df)} unique primary_ag_products.") + return transformed_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py deleted file mode 100644 index 237435cd..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/transform/products/primary_product.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional, Dict -import pandas as pd -from prefect import task, get_run_logger - -EXTRACT_SOURCES = ["basic_sample_info"] - -@task -def transform_products_primary_product(data_sources: Dict[str, pd.DataFrame]) -> Optional[pd.DataFrame]: - """ - Transforms the raw data to extract unique primary products. - - This function is purely for transformation (the 'T' in ETL). It takes the - raw DataFrame and performs the specific transformations needed for the - 'primary_product' table. - - Args: - data_sources: A dictionary of DataFrames from the extraction step. - - Returns: - A transformed DataFrame with a single 'Primary_crop' column containing - unique product names, or None if an error occurs. - """ - logger = get_run_logger() - logger.info("Transforming raw data for primary products...") - - raw_df = data_sources["basic_sample_info"] - - # Step 1: Check if the required column exists - if 'Primary_crop' not in raw_df.columns: - logger.error("'Primary_crop' column not found in the raw data.") - return None - - # Step 2: Get unique product names and create the final DataFrame - primary_product_names = raw_df['Primary_crop'].unique() - transformed_df = pd.DataFrame(primary_product_names, columns=["Primary_crop"]) - - logger.info(f"Successfully transformed data, found {len(transformed_df)} unique primary products.") - return transformed_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py new file mode 100644 index 00000000..7ae5cf9b --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_ag_product.py @@ -0,0 +1,26 @@ +from prefect import flow +from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info +from ca_biositing.pipeline.etl.transform.products.primary_ag_product import transform_products_primary_ag_product +from ca_biositing.pipeline.etl.load.products.primary_ag_product import load_products_primary_ag_product + +@flow(name="Primary Ag Product ETL", log_prints=True) +def primary_ag_product_flow(): + """ + ETL flow for processing primary agricultural products data. + + This flow extracts basic sample information, transforms it to identify + unique primary agricultural products, and loads them into the database. + """ + print("Running Primary Ag Product ETL flow...") + + # Extract + basic_sample_info_df = extract_basic_sample_info() + + # Transform + # The transform function expects a dictionary of data sources. + primary_ag_product_df = transform_products_primary_ag_product({"basic_sample_info": basic_sample_info_df}) + + # Load + load_products_primary_ag_product(primary_ag_product_df) + + print("Primary Ag Product ETL flow completed successfully.") diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py deleted file mode 100644 index 8e6a1524..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/flows/primary_product.py +++ /dev/null @@ -1,26 +0,0 @@ -from prefect import flow -from ca_biositing.pipeline.etl.extract.basic_sample_info import extract_basic_sample_info -from ca_biositing.pipeline.etl.transform.products.primary_product import transform_products_primary_product -from ca_biositing.pipeline.etl.load.products.primary_product import load_products_primary_product - -@flow(name="Primary Product ETL", log_prints=True) -def primary_product_flow(): - """ - ETL flow for processing primary products. - - This flow extracts basic sample information, transforms it to identify - unique primary products, and loads them into the database. - """ - print("Running Primary Product ETL flow...") - - # Extract - basic_sample_info_df = extract_basic_sample_info() - - # Transform - # The transform function expects a dictionary of data sources. - primary_product_df = transform_products_primary_product({"basic_sample_info": basic_sample_info_df}) - - # Load - load_products_primary_product(primary_product_df) - - print("Primary Product ETL flow completed successfully.") From fd235be591c59c3b50d6a28c1d8af3566f896075 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 30 Dec 2025 08:55:42 -0700 Subject: [PATCH 03/10] holiday work. New notebook for importing gsheeets to pandas and playing around --- .../15467b7f2f3d_test_after_revert.py | 32 + .../linkml/modules/primary_ag_product.yaml | 23 + .../linkml/modules/resource_availability.yaml | 35 + .../linkml/modules/resource_class.yaml | 21 + .../modules/resource_counterfactual.yaml | 47 + .../linkml/modules/resource_morphology.yaml | 34 + .../linkml/modules/resource_subclass.yaml | 21 + .../schemas/generated/primary_ag_product.py | 75 ++ .../generated/resource_availability.py | 308 +++++ .../schemas/generated/resource_class.py | 74 ++ .../generated/resource_counterfactual.py | 1030 +++++++++++++++++ .../schemas/generated/resource_morphology.py | 308 +++++ .../schemas/generated/resource_subclass.py | 74 ++ .../datamodels/utils/database_joins.ipynb | 491 +------- .../pipeline/etl/extract/experiments.py | 2 +- .../pipeline/etl/extract/proximate.py | 59 + .../utils/gsheet_extraction_notebook.ipynb | 179 +++ 17 files changed, 2376 insertions(+), 437 deletions(-) create mode 100644 alembic/versions/15467b7f2f3d_test_after_revert.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb diff --git a/alembic/versions/15467b7f2f3d_test_after_revert.py b/alembic/versions/15467b7f2f3d_test_after_revert.py new file mode 100644 index 00000000..38dddb9d --- /dev/null +++ b/alembic/versions/15467b7f2f3d_test_after_revert.py @@ -0,0 +1,32 @@ +"""test after revert + +Revision ID: 15467b7f2f3d +Revises: e84690898528 +Create Date: 2025-12-25 05:02:50.212384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '15467b7f2f3d' +down_revision: Union[str, Sequence[str], None] = 'e84690898528' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml new file mode 100644 index 00000000..2c5c9b4c --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml @@ -0,0 +1,23 @@ +id: https://w3id.org/ca_biositing/primary_ag_product +name: primary_ag_product +title: Primary Agricultural Product +description: Core definitions for primary agricultural products. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + PrimaryAgProduct: + is_a: LookupBase + description: Primary agricultural product definition. + slots: + - note diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml new file mode 100644 index 00000000..9bf95246 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml @@ -0,0 +1,35 @@ +id: https://w3id.org/ca_biositing/resource_availability +name: resource_availability +title: Resource Availability +description: Availability of a resource in a location. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - places + - resource_information + +classes: + ResourceAvailability: + is_a: BaseEntity + description: Availability of a resource in a location. + slots: + - resource_id + - geoid + - from_month + - to_month + - year_round + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml new file mode 100644 index 00000000..cc39fe4e --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/resource_class +name: resource_class +title: Resource Class +description: Classification of resources. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + ResourceClass: + is_a: LookupBase + description: Classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml new file mode 100644 index 00000000..d346b197 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml @@ -0,0 +1,47 @@ +id: https://w3id.org/ca_biositing/resource_counterfactual +name: resource_counterfactual +title: Resource Counterfactual +description: Counterfactual uses of a resource. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - places + - resource_information + - data_sources_metadata + +classes: + ResourceCounterfactual: + is_a: BaseEntity + description: Counterfactual uses of a resource. + slots: + - geoid + - resource_id + - counterfactual_description + - animal_bedding_percent + - animal_bedding_source_id + - animal_feed_percent + - animal_feed_source_id + - bioelectricty_percent + - bioelectricty_source_id + - burn_percent + - burn_source_id + - compost_percent + - compost_source_id + - landfill_percent + - landfill_source_id + - counterfactual_date + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml new file mode 100644 index 00000000..f9d93c67 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml @@ -0,0 +1,34 @@ +id: https://w3id.org/ca_biositing/resource_morphology +name: resource_morphology +title: Resource Morphology +description: Morphology of a resource. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + - resource_information + +classes: + ResourceMorphology: + description: Morphology of a resource. + slots: + - id + - resource_id + - morphology_uri + slot_usage: + id: + identifier: true + range: integer + +slots: + resource_id: + range: Resource + description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml new file mode 100644 index 00000000..0236bca8 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/resource_subclass +name: resource_subclass +title: Resource Subclass +description: Sub-classification of resources. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - core + +classes: + ResourceSubclass: + is_a: LookupBase + description: Sub-classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py new file mode 100644 index 00000000..dfa0f14f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py @@ -0,0 +1,75 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py new file mode 100644 index 00000000..c02c738f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py @@ -0,0 +1,308 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py new file mode 100644 index 00000000..b47201f8 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py @@ -0,0 +1,74 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py new file mode 100644 index 00000000..4568cca1 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py @@ -0,0 +1,1030 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class ParameterCategoryParameter(Base): + """ + Link between Parameter and ParameterCategory. + """ + __tablename__ = 'parameter_category_parameter' + + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer()) + parameter_category_id = Column(Integer()) + + + def __repr__(self): + return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" + + + + + + +class ParameterUnit(Base): + """ + Link between Parameter and Unit (alternate units). + """ + __tablename__ = 'parameter_unit' + + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer()) + alternate_unit_id = Column(Integer()) + + + def __repr__(self): + return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" + + + + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Contact(BaseEntity): + """ + Contact information for a person. + """ + __tablename__ = 'contact' + + first_name = Column(Text()) + last_name = Column(Text()) + email = Column(Text()) + affiliation = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Provider(BaseEntity): + """ + Provider information. + """ + __tablename__ = 'provider' + + codename = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class DataSource(BaseEntity): + """ + Source of data. + """ + __tablename__ = 'data_source' + + name = Column(Text()) + description = Column(Text()) + data_source_type_id = Column(Integer()) + full_title = Column(Text()) + creator = Column(Text()) + subject = Column(Text()) + publisher = Column(Text()) + contributor = Column(Text()) + date = Column(DateTime()) + type = Column(Text()) + biocirv = Column(Boolean()) + format = Column(Text()) + language = Column(Text()) + relation = Column(Text()) + temporal_coverage = Column(Text()) + location_coverage_id = Column(Integer()) + rights = Column(Text()) + license = Column(Text()) + uri = Column(Text()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FileObjectMetadata(BaseEntity): + """ + Metadata for a file object. + """ + __tablename__ = 'file_object_metadata' + + data_source_id = Column(Integer()) + bucket_path = Column(Text()) + file_format = Column(Text()) + file_size = Column(Integer()) + checksum_md5 = Column(Text()) + checksum_sha256 = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class DataSourceType(BaseEntity): + """ + Type of data source (e.g. database, literature). + """ + __tablename__ = 'data_source_type' + + source_type_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationResolution(LookupBase): + """ + Resolution of the location (e.g. nation, state, county). + """ + __tablename__ = 'location_resolution' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class SourceType(LookupBase): + """ + Type of source (e.g. database, literature). + """ + __tablename__ = 'source_type' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Unit(LookupBase): + """ + Unit of measurement. + """ + __tablename__ = 'unit' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Method(BaseEntity): + """ + Analytical method. + """ + __tablename__ = 'method' + + name = Column(Text()) + method_abbrev_id = Column(Integer()) + method_category_id = Column(Integer()) + method_standard_id = Column(Integer()) + description = Column(Text()) + detection_limits = Column(Text()) + source_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodAbbrev(LookupBase): + """ + Abbreviation for method. + """ + __tablename__ = 'method_abbrev' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodCategory(LookupBase): + """ + Category of method. + """ + __tablename__ = 'method_category' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class MethodStandard(LookupBase): + """ + Standard associated with the method. + """ + __tablename__ = 'method_standard' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Parameter(BaseEntity): + """ + Parameter being measured. + """ + __tablename__ = 'parameter' + + name = Column(Text()) + standard_unit_id = Column(Integer()) + calculated = Column(Boolean()) + description = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ParameterCategory(LookupBase): + """ + Category of parameter. + """ + __tablename__ = 'parameter_category' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldSample(BaseEntity): + """ + Sample collected from the field. + """ + __tablename__ = 'field_sample' + + name = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + provider_id = Column(Integer()) + collector_id = Column(Integer()) + sample_collection_source = Column(Text()) + amount_collected = Column(Numeric()) + amount_collected_unit_id = Column(Integer()) + sampling_location_id = Column(Integer()) + field_storage_method_id = Column(Integer()) + field_storage_duration_value = Column(Numeric()) + field_storage_duration_unit_id = Column(Integer()) + field_storage_location_id = Column(Integer()) + collection_timestamp = Column(DateTime()) + collection_method_id = Column(Integer()) + harvest_method_id = Column(Integer()) + harvest_date = Column(Date()) + field_sample_storage_location_id = Column(Integer()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PhysicalCharacteristic(BaseEntity): + """ + Physical characteristics of a sample. + """ + __tablename__ = 'physical_characteristic' + + field_sample_id = Column(Integer()) + particle_length = Column(Numeric()) + particle_width = Column(Numeric()) + particle_height = Column(Numeric()) + particle_unit_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldSampleCondition(BaseEntity): + """ + Condition of the field sample. + """ + __tablename__ = 'field_sample_condition' + + field_sample_id = Column(Integer()) + ag_treatment_id = Column(Integer()) + last_application_date = Column(Date()) + treatment_amount_per_acre = Column(Float()) + processing_method_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class FieldStorageMethod(LookupBase): + """ + Method of field storage. + """ + __tablename__ = 'field_storage_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class CollectionMethod(LookupBase): + """ + Method of collection. + """ + __tablename__ = 'collection_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class HarvestMethod(LookupBase): + """ + Method of harvest. + """ + __tablename__ = 'harvest_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ProcessingMethod(LookupBase): + """ + Method of processing. + """ + __tablename__ = 'processing_method' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class SoilType(LookupBase): + """ + Type of soil. + """ + __tablename__ = 'soil_type' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class AgTreatment(LookupBase): + """ + Agricultural treatment. + """ + __tablename__ = 'ag_treatment' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationSoilType(BaseEntity): + """ + Soil type at a location. + """ + __tablename__ = 'location_soil_type' + + location_id = Column(Integer()) + soil_type_id = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py new file mode 100644 index 00000000..0b9bcf94 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py @@ -0,0 +1,308 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class ResourceMorphology(Base): + """ + Morphology of a resource. + """ + __tablename__ = 'resource_morphology' + + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) + + + def __repr__(self): + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" + + + + + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class Geography(Base): + """ + Geographic location. + """ + __tablename__ = 'geography' + + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) + + + def __repr__(self): + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" + + + + + + +class LocationAddress(BaseEntity): + """ + Physical address. + """ + __tablename__ = 'location_address' + + geography_id = Column(Text()) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class Resource(BaseEntity): + """ + Biomass resource definition. + """ + __tablename__ = 'resource' + + name = Column(Text()) + primary_ag_product_id = Column(Integer()) + resource_class_id = Column(Integer()) + resource_subclass_id = Column(Integer()) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceClass(LookupBase): + """ + Classification of resources. + """ + __tablename__ = 'resource_class' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class PrimaryAgProduct(LookupBase): + """ + Primary agricultural product definition. + """ + __tablename__ = 'primary_ag_product' + + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceAvailability(BaseEntity): + """ + Availability of a resource in a location. + """ + __tablename__ = 'resource_availability' + + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class ResourceCounterfactual(BaseEntity): + """ + Counterfactual uses of a resource. + """ + __tablename__ = 'resource_counterfactual' + + geoid = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + counterfactual_description = Column(Text()) + animal_bedding_percent = Column(Numeric()) + animal_bedding_source_id = Column(Integer()) + animal_feed_percent = Column(Numeric()) + animal_feed_source_id = Column(Integer()) + bioelectricty_percent = Column(Numeric()) + bioelectricty_source_id = Column(Integer()) + burn_percent = Column(Numeric()) + burn_source_id = Column(Integer()) + compost_percent = Column(Numeric()) + compost_source_id = Column(Integer()) + landfill_percent = Column(Numeric()) + landfill_source_id = Column(Integer()) + counterfactual_date = Column(Date()) + note = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py new file mode 100644 index 00000000..a3919852 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py @@ -0,0 +1,74 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class BaseEntity(Base): + """ + Base entity included in all main entity tables. + """ + __tablename__ = 'base_entity' + + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + + +class LookupBase(Base): + """ + Base class for enum/ontology-like tables. + """ + __tablename__ = 'lookup_base' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + + +class ResourceSubclass(LookupBase): + """ + Sub-classification of resources. + """ + __tablename__ = 'resource_subclass' + + id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) + + + def __repr__(self): + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } diff --git a/src/ca_biositing/datamodels/utils/database_joins.ipynb b/src/ca_biositing/datamodels/utils/database_joins.ipynb index dbb836f2..e4a9b9a5 100644 --- a/src/ca_biositing/datamodels/utils/database_joins.ipynb +++ b/src/ca_biositing/datamodels/utils/database_joins.ipynb @@ -15,21 +15,13 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Python Executable: /Users/pjsmitty301/ca-biositing/.pixi/envs/default/bin/python\n", - "✅ You are running in a Pixi environment.\n" - ] - } - ], + "outputs": [], "source": [ "# Verify Kernel\n", "import sys\n", + "\n", "print(f\"Python Executable: {sys.executable}\")\n", "\n", "# Check if we are in the pixi environment (path should contain .pixi)\n", @@ -41,17 +33,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Libraries and models imported successfully.\n" - ] - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "from sqlalchemy import create_engine, select\n", @@ -65,17 +49,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected to database.\n" - ] - } - ], + "outputs": [], "source": [ "# Database Connection\n", "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", @@ -96,22 +72,23 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": " is not a generic class", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m stmt2 = select(LandiqRecord, PrimaryCrop).join(\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mPrimaryCrop\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m, \n\u001b[32m 3\u001b[39m LandiqRecord.main_crop == PrimaryCrop.id).limit(\u001b[32m10\u001b[39m)\n\u001b[32m 5\u001b[39m \u001b[38;5;28mprint\u001b[39m(stmt2)\n\u001b[32m 7\u001b[39m df2 = pd.read_sql(stmt2, engine)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/typing.py:398\u001b[39m, in \u001b[36m_tp_cache..decorator..inner\u001b[39m\u001b[34m(*args, **kwds)\u001b[39m\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[32m 397\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m \u001b[38;5;66;03m# All real errors (not unhashable args) are raised below.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m398\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/typing.py:1110\u001b[39m, in \u001b[36m_generic_class_getitem\u001b[39m\u001b[34m(cls, params)\u001b[39m\n\u001b[32m 1108\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m prepare \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1109\u001b[39m params = prepare(\u001b[38;5;28mcls\u001b[39m, params)\n\u001b[32m-> \u001b[39m\u001b[32m1110\u001b[39m \u001b[43m_check_generic\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m__parameters__\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1112\u001b[39m new_args = []\n\u001b[32m 1113\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m param, new_arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;28mcls\u001b[39m.__parameters__, params):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/ca-biositing/.pixi/envs/default/lib/python3.12/site-packages/typing_extensions.py:3111\u001b[39m, in \u001b[36m_check_generic\u001b[39m\u001b[34m(cls, parameters, elen)\u001b[39m\n\u001b[32m 3106\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"Check correct count for parameters of a generic cls (internal helper).\u001b[39;00m\n\u001b[32m 3107\u001b[39m \n\u001b[32m 3108\u001b[39m \u001b[33;03mThis gives a nice error message in case of count mismatch.\u001b[39;00m\n\u001b[32m 3109\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 3110\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m elen:\n\u001b[32m-> \u001b[39m\u001b[32m3111\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is not a generic class\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 3112\u001b[39m alen = \u001b[38;5;28mlen\u001b[39m(parameters)\n\u001b[32m 3113\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m alen != elen:\n", - "\u001b[31mTypeError\u001b[39m: is not a generic class" - ] - } - ], + "outputs": [], + "source": [ + "# Create the join query\n", + "stmt = select(LandiqRecord, Observation).join(\n", + " Observation,\n", + " Observation.record_id == LandiqRecord.id\n", + ")\n", + "\n", + "print(\"SQL Query:\")\n", + "print(stmt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "stmt2 = select(LandiqRecord, PrimaryCrop).join(\n", " PrimaryCrop, \n", @@ -124,281 +101,11 @@ "df2.columns\n" ] }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SQL Query:\n", - "SELECT landiq_record.dataset_id, landiq_record.polygon_id, landiq_record.main_crop, landiq_record.secondary_crop, landiq_record.tertiary_crop, landiq_record.quaternary_crop, landiq_record.confidence, landiq_record.irrigated, landiq_record.acres, landiq_record.version, landiq_record.note, landiq_record.test, landiq_record.id, landiq_record.created_at, landiq_record.updated_at, landiq_record.etl_run_id, landiq_record.lineage_group_id, observation.dataset_id AS dataset_id_1, observation.record_type, observation.record_id, observation.parameter_id, observation.value, observation.unit_id, observation.dimension_type_id, observation.dimension_value, observation.dimension_unit_id, observation.note AS note_1, observation.id AS id_1, observation.created_at AS created_at_1, observation.updated_at AS updated_at_1, observation.etl_run_id AS etl_run_id_1, observation.lineage_group_id AS lineage_group_id_1 \n", - "FROM landiq_record JOIN observation ON observation.record_id = landiq_record.id\n" - ] - } - ], - "source": [ - "# Create the join query\n", - "stmt = select(LandiqRecord, Observation).join(\n", - " Observation,\n", - " Observation.record_id == LandiqRecord.id\n", - ")\n", - "\n", - "print(\"SQL Query:\")\n", - "print(stmt)" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded 0 rows.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "dataset_id", - "rawType": "object", - "type": "string" - }, - { - "name": "polygon_id", - "rawType": "object", - "type": "string" - }, - { - "name": "main_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "secondary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "tertiary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "quaternary_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "confidence", - "rawType": "object", - "type": "string" - }, - { - "name": "irrigated", - "rawType": "object", - "type": "string" - }, - { - "name": "acres", - "rawType": "object", - "type": "string" - }, - { - "name": "version", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "string" - }, - { - "name": "test", - "rawType": "object", - "type": "string" - }, - { - "name": "id", - "rawType": "object", - "type": "string" - }, - { - "name": "created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "etl_run_id", - "rawType": "object", - "type": "string" - }, - { - "name": "lineage_group_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dataset_id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "record_type", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "object", - "type": "string" - }, - { - "name": "unit_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_type_id", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_value", - "rawType": "object", - "type": "string" - }, - { - "name": "dimension_unit_id", - "rawType": "object", - "type": "string" - }, - { - "name": "note_1", - "rawType": "object", - "type": "string" - }, - { - "name": "id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "created_at_1", - "rawType": "object", - "type": "string" - }, - { - "name": "updated_at_1", - "rawType": "object", - "type": "string" - }, - { - "name": "etl_run_id_1", - "rawType": "object", - "type": "string" - }, - { - "name": "lineage_group_id_1", - "rawType": "object", - "type": "string" - } - ], - "ref": "edc059e1-7169-4eb9-ad76-c55b46d93053", - "rows": [], - "shape": { - "columns": 32, - "rows": 0 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dataset_idpolygon_idmain_cropsecondary_croptertiary_cropquaternary_cropconfidenceirrigatedacresversion...unit_iddimension_type_iddimension_valuedimension_unit_idnote_1id_1created_at_1updated_at_1etl_run_id_1lineage_group_id_1
\n", - "

0 rows × 32 columns

\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [dataset_id, polygon_id, main_crop, secondary_crop, tertiary_crop, quaternary_crop, confidence, irrigated, acres, version, note, test, id, created_at, updated_at, etl_run_id, lineage_group_id, dataset_id_1, record_type, record_id, parameter_id, value, unit_id, dimension_type_id, dimension_value, dimension_unit_id, note_1, id_1, created_at_1, updated_at_1, etl_run_id_1, lineage_group_id_1]\n", - "Index: []\n", - "\n", - "[0 rows x 32 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Execute and load into DataFrame\n", "try:\n", @@ -415,135 +122,47 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['dataset_id', 'polygon_id', 'main_crop', 'secondary_crop',\n", - " 'tertiary_crop', 'quaternary_crop', 'confidence', 'irrigated', 'acres',\n", - " 'version', 'note', 'test', 'id', 'created_at', 'updated_at',\n", - " 'etl_run_id', 'lineage_group_id', 'dataset_id_1', 'record_type',\n", - " 'record_id', 'parameter_id', 'value', 'unit_id', 'dimension_type_id',\n", - " 'dimension_value', 'dimension_unit_id', 'note_1', 'id_1',\n", - " 'created_at_1', 'updated_at_1', 'etl_run_id_1', 'lineage_group_id_1'],\n", - " dtype='object')" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.columns" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "id", - "rawType": "object", - "type": "string" - }, - { - "name": "dataset_id", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "main_crop", - "rawType": "object", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "object", - "type": "string" - }, - { - "name": "unit_id", - "rawType": "object", - "type": "string" - } - ], - "ref": "5c95c323-56ec-4546-ad20-84e7b50c7748", - "rows": [], - "shape": { - "columns": 7, - "rows": 0 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
iddataset_idrecord_idmain_cropparameter_idvalueunit_id
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [id, dataset_id, record_id, main_crop, parameter_id, value, unit_id]\n", - "Index: []" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[[\"id\", \"dataset_id\", \"record_id\", \"main_crop\", \"parameter_id\", \"value\", \"unit_id\"]]" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = select(Observation, LandiqRecord, PrimaryCrop).join(\n", + " Observation,\n", + " Observation.record_id == LandiqRecord.id).join(\n", + " PrimaryCrop,\n", + " LandiqRecord.main_crop == PrimaryCrop.id).limit(10)\n", + "\n", + "df3 = pd.read_sql(query, engine)\n", + "\n", + "df3.columns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.read_sql(select(LandiqRecord), engine)" + ] } ], "metadata": { diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py index 25a6d194..9a74e350 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/experiments.py @@ -1,7 +1,7 @@ from typing import Optional import pandas as pd from prefect import task, get_run_logger -from ca_biositing.pipeline.utils.gsheet_to_pandas import gsheet_to_df +from ...utils.gsheet_to_pandas import gsheet_to_df @task def extract_experiments() -> Optional[pd.DataFrame]: diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py new file mode 100644 index 00000000..9bbaf910 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/proximate.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.1-Proximate" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb new file mode 100644 index 00000000..b098f40d --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import janitor as jn\n", + "from IPython.display import display\n", + "\n", + "# --- Robustly find the project root ---\n", + "# The project root is the directory containing the 'pixi.toml' file.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path): # Stop at the filesystem root\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# --- Add project root to sys.path ---\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " print(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " print(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import the module ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate\n", + " print(\"Successfully imported 'proximate' module.\")\n", + "except ImportError as e:\n", + " print(f\"Failed to import 'proximate' module: {e}\")\n", + " print(f\"\\nFull sys.path: {sys.path}\")\n", + "\n", + "# --- Run the extraction ---\n", + "if 'proximate' in locals():\n", + " try:\n", + " # Pass the project_root to the extract function\n", + " df = proximate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted data.\")\n", + " display(df.head())\n", + " else:\n", + " print(\"\\nExtraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during extraction: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.clean_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df[['Record_ID', 'Source_codename', 'Prepared_sample']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Convert 'Value' column to numeric\n", + "pv_df = df[['Parameter', 'Value']]\n", + "\n", + "pv_df['Value'] = pd.to_numeric(pv_df['Value'], errors='coerce')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pv_df\n", + "\n", + "if pv_df.groupby('Parameter').mean() is not None:\n", + " print(\"Mean values by Parameter:\")\n", + " display(pv_df.groupby('Parameter').mean())\n", + "else:\n", + " print(\"No numeric values available to compute means.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = df[['Resource', 'Parameter', 'Value', 'Unit']]\n", + "\n", + "df2['Value'] = pd.to_numeric(df2['Value'], errors='coerce')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = df2.dropna(subset=['Value'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats = df2.groupby(['Resource', 'Parameter'])['Value'].agg(['mean', 'median', 'min', 'max', 'std', 'count'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 907686ba1ca492efd578895d1eb825ea9e7ea961 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Fri, 2 Jan 2026 21:41:29 -0700 Subject: [PATCH 04/10] feat: notebook for gsheets extraction, data playground, and pk id lookup insertion. --- .../pipeline/etl/extract/cmpana.py | 59 + .../pipeline/etl/extract/ultimate.py | 59 + .../ca_biositing/pipeline/utils/engine.py | 6 + .../utils/gsheet_extraction_notebook.ipynb | 4100 ++++++++++++++++- .../pipeline/utils/name_id_swap.py | 81 + .../pipeline/utils/seed_biomass.sql | 15 - 6 files changed, 4272 insertions(+), 48 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py delete mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py new file mode 100644 index 00000000..b34a39ca --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/cmpana.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.3-CmpAna" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py new file mode 100644 index 00000000..c7495876 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/etl/extract/ultimate.py @@ -0,0 +1,59 @@ +""" +ETL Extract Template +--- + +This module provides a template for extracting data from a Google Sheet. + +To use this template: +1. Copy this file to the appropriate subdirectory in `src/etl/extract/`. + For example: `src/etl/extract/new_module/new_data.py` +2. Update the configuration constants (`GSHEET_NAME`, `WORKSHEET_NAME`). +3. Ensure the `CREDENTIALS_PATH` is correct. +""" + +from typing import Optional +import os +import pandas as pd +from prefect import task, get_run_logger +from ...utils.gsheet_to_pandas import gsheet_to_df + +# --- CONFIGURATION --- +# TODO: Replace with the name of the Google Sheet file. +GSHEET_NAME = "Aim 1-Feedstock Collection and Processing Data-BioCirV" + +# TODO: Replace with the exact name of the worksheet/tab to extract data from. +WORKSHEET_NAME = "03.7-Ultimate" + +# The path to the credentials file. This is typically kept in the project root. +CREDENTIALS_PATH = "credentials.json" + + +@task +def extract(project_root: Optional[str] = None) -> Optional[pd.DataFrame]: + """ + Extracts raw data from the specified Google Sheet worksheet. + + This function serves as the 'Extract' step in an ETL pipeline. It connects + to the data source and returns the data as is, without transformation. + + Returns: + A pandas DataFrame containing the raw data, or None if an error occurs. + """ + logger = get_run_logger() + logger.info(f"Extracting raw data from '{WORKSHEET_NAME}' in '{GSHEET_NAME}'...") + + # If project_root is provided (e.g., from a notebook), construct an absolute path + # Otherwise, use the default relative path (for the main pipeline) + credentials_path = CREDENTIALS_PATH + if project_root: + credentials_path = os.path.join(project_root, CREDENTIALS_PATH) + + # The gsheet_to_df function handles authentication, data fetching, and error handling. + raw_df = gsheet_to_df(GSHEET_NAME, WORKSHEET_NAME, credentials_path) + + if raw_df is None: + logger.error("Failed to extract data. Aborting.") + return None + + logger.info("Successfully extracted raw data.") + return raw_df diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py new file mode 100644 index 00000000..a24e92a0 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py @@ -0,0 +1,6 @@ +from sqlmodel import create_engine, Session + +DATABASE_URL = "postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db" +engine = create_engine(DATABASE_URL) + +db_session = Session(engine) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb index b098f40d..9d27799e 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -2,13 +2,1522 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", + "Successfully imported all module.\n" + ] + }, + { + "data": { + "text/html": [ + "
20:54:42.753 | INFO    | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:42.753 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.230 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.230 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.234 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.234 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted proximate data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Prox_UUID_031", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Source_codename", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + } + ], + "ref": "b6c1f313-06e4-4dba-8a9e-6a1dc878820f", + "rows": [ + [ + "0", + "D7965110-407F-E356-D41D-B3B9A2B7B7", + "(73)B7B7", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "1", + "Prox01xk(73)1", + "Moisture", + "61.85", + "% total weight", + "2024-10-02 10:31:01", + "", + "Pass", + "not ready", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "1", + "C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8", + "(73)9BA8", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "2", + "Prox01xk(73)2", + "Moisture", + "63.21", + "% total weight", + "2024-10-02 10:31:31", + "", + "Pass", + "ready", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "2", + "DF304D5D-3A85-4881-7142-6D4E5F957D", + "(73)957D", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "3", + "Prox01xk(73)3", + "Moisture", + "63.27", + "% total weight", + "2024-10-02 10:32:01", + "", + "Pass", + "imported", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "3", + "01C6C5BE-CEA6-54AF-3924-B0BAD69335", + "(73)9335", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "1", + "Prox01xk(73)1", + "Ash", + "0.69", + "% total weight", + "2024-10-03 10:31:01", + "", + "Pass", + "import failed", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ], + [ + "4", + "126745C7-DD41-2F6D-0DC5-28DBCA415F", + "(73)415F", + "Oakleaf", + "Oak-TmPm01A(73)", + "Tomato pomace", + "As Is", + "4C", + "Prox01xk", + "2", + "Prox01xk(73)2", + "Ash", + "0.89", + "% total weight", + "2024-10-03 10:31:31", + "", + "Pass", + "", + "", + "Proximate analysis", + "xkang2@lbl.gov" + ] + ], + "shape": { + "columns": 20, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Prox_UUID_031Record_IDSource_codenamePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameterValueUnitCreated_atUpdated_atQC_resultUpload_statusNoteAnalysis_typeAnalyst_email
0D7965110-407F-E356-D41D-B3B9A2B7B7(73)B7B7OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk1Prox01xk(73)1Moisture61.85% total weight2024-10-02 10:31:01Passnot readyProximate analysisxkang2@lbl.gov
1C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8(73)9BA8OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk2Prox01xk(73)2Moisture63.21% total weight2024-10-02 10:31:31PassreadyProximate analysisxkang2@lbl.gov
2DF304D5D-3A85-4881-7142-6D4E5F957D(73)957DOakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk3Prox01xk(73)3Moisture63.27% total weight2024-10-02 10:32:01PassimportedProximate analysisxkang2@lbl.gov
301C6C5BE-CEA6-54AF-3924-B0BAD69335(73)9335OakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk1Prox01xk(73)1Ash0.69% total weight2024-10-03 10:31:01Passimport failedProximate analysisxkang2@lbl.gov
4126745C7-DD41-2F6D-0DC5-28DBCA415F(73)415FOakleafOak-TmPm01A(73)Tomato pomaceAs Is4CProx01xk2Prox01xk(73)2Ash0.89% total weight2024-10-03 10:31:31PassProximate analysisxkang2@lbl.gov
\n", + "
" + ], + "text/plain": [ + " Prox_UUID_031 Record_ID Source_codename \\\n", + "0 D7965110-407F-E356-D41D-B3B9A2B7B7 (73)B7B7 Oakleaf \n", + "1 C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8 (73)9BA8 Oakleaf \n", + "2 DF304D5D-3A85-4881-7142-6D4E5F957D (73)957D Oakleaf \n", + "3 01C6C5BE-CEA6-54AF-3924-B0BAD69335 (73)9335 Oakleaf \n", + "4 126745C7-DD41-2F6D-0DC5-28DBCA415F (73)415F Oakleaf \n", + "\n", + " Prepared_sample Resource Preparation_method Storage_cond \\\n", + "0 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "1 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "2 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "3 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "4 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", + "\n", + " Exper_abbrev Repl_no Repl_ID Parameter Value Unit \\\n", + "0 Prox01xk 1 Prox01xk(73)1 Moisture 61.85 % total weight \n", + "1 Prox01xk 2 Prox01xk(73)2 Moisture 63.21 % total weight \n", + "2 Prox01xk 3 Prox01xk(73)3 Moisture 63.27 % total weight \n", + "3 Prox01xk 1 Prox01xk(73)1 Ash 0.69 % total weight \n", + "4 Prox01xk 2 Prox01xk(73)2 Ash 0.89 % total weight \n", + "\n", + " Created_at Updated_at QC_result Upload_status Note \\\n", + "0 2024-10-02 10:31:01 Pass not ready \n", + "1 2024-10-02 10:31:31 Pass ready \n", + "2 2024-10-02 10:32:01 Pass imported \n", + "3 2024-10-03 10:31:01 Pass import failed \n", + "4 2024-10-03 10:31:31 Pass \n", + "\n", + " Analysis_type Analyst_email \n", + "0 Proximate analysis xkang2@lbl.gov \n", + "1 Proximate analysis xkang2@lbl.gov \n", + "2 Proximate analysis xkang2@lbl.gov \n", + "3 Proximate analysis xkang2@lbl.gov \n", + "4 Proximate analysis xkang2@lbl.gov " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:44.272 | INFO    | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:44.272 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.593 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.593 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.595 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.595 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted Ultimate data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Ult_UUID_037", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Ult_sample_name", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "Raw_data_URL", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + } + ], + "ref": "61714e59-0541-4b74-9d38-a37794cbd34d", + "rows": [ + [ + "0", + "421617A5-E50E-9642-2974-B3275FE822", + ")U22E822", + "Hum-AlmHu023KM2(15)U22", + "Hum-AlmHu023KM2(15)", + "Almond Hulls", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(15)1", + "DM", + "8.86E+01", + "pc", + "", + "", + "Pass", + "1", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "1", + "7E7919C2-5DB4-6BEF-75E2-7E51321200", + ")U001200", + "Hum-AlmHu023KM2(15)U00", + "Hum-AlmHu023KM2(15)", + "Almond Hulls", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(15)1", + "DM", + "", + "pc", + "", + "", + "Fail", + " 1 dup", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "2", + "3AA85881-1185-642F-C44B-41AD2275D2", + ")UD275D2", + "Hum-AlmSh022KM2(13)UD2", + "Hum-AlmSh022KM2(13)", + "Almond Shells", + "Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(13)1", + "DM", + "8.13E+01", + "pc", + "", + "", + "Pass", + "2", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "3", + "FA418804-6C4F-4C90-D78F-84D7DF54D3", + ")UD354D3", + "Ene-WaSh017OKM2(82)UD3", + "Ene-WaSh017OKM2(82)", + "Walnut Shells", + "Oven Dry + Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(82)1", + "DM", + "9.22E+01", + "pc", + "", + "", + "Pass", + "3", + "Ultimate analysis", + "", + "", + "", + "" + ], + [ + "4", + "6FDACBFC-7E0B-473B-444F-85B7650267", + ")U670267", + "Ebo-GpPm010OKM2(1B)U67", + "Ebo-GpPm010OKM2(1B)", + "Grape pomace", + "Oven Dry + Knife Mill (2mm)", + "RT vacuum sealed", + "Ult26kh", + "1", + "Ult26kh(1B)1", + "DM", + "9.38E+01", + "pc", + "", + "", + "Pass", + "4", + "Ultimate analysis", + "", + "", + "", + "" + ] + ], + "shape": { + "columns": 22, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Ult_UUID_037Record_IDUlt_sample_namePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_ID...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
0421617A5-E50E-9642-2974-B3275FE822)U22E822Hum-AlmHu023KM2(15)U22Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcPass1Ultimate analysis
17E7919C2-5DB4-6BEF-75E2-7E51321200)U001200Hum-AlmHu023KM2(15)U00Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcFail1 dupUltimate analysis
23AA85881-1185-642F-C44B-41AD2275D2)UD275D2Hum-AlmSh022KM2(13)UD2Hum-AlmSh022KM2(13)Almond ShellsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(13)1...pcPass2Ultimate analysis
3FA418804-6C4F-4C90-D78F-84D7DF54D3)UD354D3Ene-WaSh017OKM2(82)UD3Ene-WaSh017OKM2(82)Walnut ShellsOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(82)1...pcPass3Ultimate analysis
46FDACBFC-7E0B-473B-444F-85B7650267)U670267Ebo-GpPm010OKM2(1B)U67Ebo-GpPm010OKM2(1B)Grape pomaceOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(1B)1...pcPass4Ultimate analysis
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " Ult_UUID_037 Record_ID Ult_sample_name \\\n", + "0 421617A5-E50E-9642-2974-B3275FE822 )U22E822 Hum-AlmHu023KM2(15)U22 \n", + "1 7E7919C2-5DB4-6BEF-75E2-7E51321200 )U001200 Hum-AlmHu023KM2(15)U00 \n", + "2 3AA85881-1185-642F-C44B-41AD2275D2 )UD275D2 Hum-AlmSh022KM2(13)UD2 \n", + "3 FA418804-6C4F-4C90-D78F-84D7DF54D3 )UD354D3 Ene-WaSh017OKM2(82)UD3 \n", + "4 6FDACBFC-7E0B-473B-444F-85B7650267 )U670267 Ebo-GpPm010OKM2(1B)U67 \n", + "\n", + " Prepared_sample Resource Preparation_method \\\n", + "0 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", + "1 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", + "2 Hum-AlmSh022KM2(13) Almond Shells Knife Mill (2mm) \n", + "3 Ene-WaSh017OKM2(82) Walnut Shells Oven Dry + Knife Mill (2mm) \n", + "4 Ebo-GpPm010OKM2(1B) Grape pomace Oven Dry + Knife Mill (2mm) \n", + "\n", + " Storage_cond Exper_abbrev Repl_no Repl_ID ... Unit Created_at \\\n", + "0 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", + "1 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", + "2 RT vacuum sealed Ult26kh 1 Ult26kh(13)1 ... pc \n", + "3 RT vacuum sealed Ult26kh 1 Ult26kh(82)1 ... pc \n", + "4 RT vacuum sealed Ult26kh 1 Ult26kh(1B)1 ... pc \n", + "\n", + " Updated_at QC_result Note Analysis_type Equipment Raw_data_URL \\\n", + "0 Pass 1 Ultimate analysis \n", + "1 Fail 1 dup Ultimate analysis \n", + "2 Pass 2 Ultimate analysis \n", + "3 Pass 3 Ultimate analysis \n", + "4 Pass 4 Ultimate analysis \n", + "\n", + " Analyst_email Upload_status \n", + "0 \n", + "1 \n", + "2 \n", + "3 \n", + "4 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:45.631 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:54:45.631 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:47.211 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:54:47.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:54:47.214 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:54:47.214 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Successfully extracted CmpAna data.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "Cmp_UUID_033", + "rawType": "object", + "type": "string" + }, + { + "name": "Record_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "Resource", + "rawType": "object", + "type": "string" + }, + { + "name": "Preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "Storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "Exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_no", + "rawType": "object", + "type": "string" + }, + { + "name": "Repl_ID", + "rawType": "object", + "type": "string" + }, + { + "name": "Parameter", + "rawType": "object", + "type": "string" + }, + { + "name": "Value", + "rawType": "object", + "type": "string" + }, + { + "name": "Unit", + "rawType": "object", + "type": "string" + }, + { + "name": "Created_at", + "rawType": "object", + "type": "string" + }, + { + "name": "Updated_at", + "rawType": "object", + "type": "string" + }, + { + "name": "QC_result", + "rawType": "object", + "type": "string" + }, + { + "name": "Note", + "rawType": "object", + "type": "string" + }, + { + "name": "Analysis_type", + "rawType": "object", + "type": "string" + }, + { + "name": "Equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "Raw_data_URL", + "rawType": "object", + "type": "string" + }, + { + "name": "Analyst_email", + "rawType": "object", + "type": "string" + }, + { + "name": "Upload_status", + "rawType": "object", + "type": "string" + } + ], + "ref": "2dd03f9d-30e4-4e23-af07-409bfdca5047", + "rows": [ + [ + "0", + "3EE2993D-86E3-1F16-C7EA-F8D555E114", + "(85)E114", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "Glucan", + "14.16", + "% dry weight", + "1/23/2025 9:00:01", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "1", + "46878EF9-1226-22A0-D5D8-CF65E241CB", + "(85)41CB", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "Glucan", + "14.18", + "% dry weight", + "1/23/2025 9:00:16", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "2", + "76A7A2F4-C4E4-E60F-1187-DEC6E02246", + "(85)2246", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "Glucan", + "14.12", + "% dry weight", + "1/23/2025 9:00:31", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "3", + "7A136832-286B-07CB-62DE-ACF52F9311", + "(85)9311", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "Glucose", + "15.74", + "% dry weight", + "1/23/2025 9:00:46", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ], + [ + "4", + "B709ECEE-F9A6-A55D-A59E-93B7B863D7", + "(85)63D7", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "Glucose", + "15.75", + "% dry weight", + "1/23/2025 9:01:01", + "", + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready" + ] + ], + "shape": { + "columns": 21, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Cmp_UUID_033Record_IDPrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameter...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucan...% dry weight1/23/2025 9:00:01passChemical compositionxkang2@lbl.govready
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucan...% dry weight1/23/2025 9:00:16passChemical compositionxkang2@lbl.govready
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)3Glucan...% dry weight1/23/2025 9:00:31passChemical compositionxkang2@lbl.govready
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucose...% dry weight1/23/2025 9:00:46passChemical compositionxkang2@lbl.govready
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucose...% dry weight1/23/2025 9:01:01passChemical compositionxkang2@lbl.govready
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " Cmp_UUID_033 Record_ID Prepared_sample \\\n", + "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", + "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", + "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", + "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", + "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", + "\n", + " Resource Preparation_method Storage_cond Exper_abbrev Repl_no \\\n", + "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", + "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", + "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 3 \n", + "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", + "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", + "\n", + " Repl_ID Parameter ... Unit Created_at Updated_at \\\n", + "0 Cmp04xk(85)1 Glucan ... % dry weight 1/23/2025 9:00:01 \n", + "1 Cmp04xk(85)2 Glucan ... % dry weight 1/23/2025 9:00:16 \n", + "2 Cmp04xk(85)3 Glucan ... % dry weight 1/23/2025 9:00:31 \n", + "3 Cmp04xk(85)1 Glucose ... % dry weight 1/23/2025 9:00:46 \n", + "4 Cmp04xk(85)2 Glucose ... % dry weight 1/23/2025 9:01:01 \n", + "\n", + " QC_result Note Analysis_type Equipment Raw_data_URL Analyst_email \\\n", + "0 pass Chemical composition xkang2@lbl.gov \n", + "1 pass Chemical composition xkang2@lbl.gov \n", + "2 pass Chemical composition xkang2@lbl.gov \n", + "3 pass Chemical composition xkang2@lbl.gov \n", + "4 pass Chemical composition xkang2@lbl.gov \n", + "\n", + " Upload_status \n", + "0 ready \n", + "1 ready \n", + "2 ready \n", + "3 ready \n", + "4 ready \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Cmp Ana extraction process completed.\n" + ] + } + ], "source": [ "import os\n", "import sys\n", "import pandas as pd\n", + "import numpy as np\n", "import janitor as jn\n", "from IPython.display import display\n", "\n", @@ -34,10 +1543,10 @@ "\n", "# --- Import the module ---\n", "try:\n", - " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate\n", - " print(\"Successfully imported 'proximate' module.\")\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " print(\"Successfully imported all module.\")\n", "except ImportError as e:\n", - " print(f\"Failed to import 'proximate' module: {e}\")\n", + " print(f\"Failed to import modules: {e}\")\n", " print(f\"\\nFull sys.path: {sys.path}\")\n", "\n", "# --- Run the extraction ---\n", @@ -46,21 +1555,67 @@ " # Pass the project_root to the extract function\n", " df = proximate.extract(project_root=project_root)\n", " if df is not None:\n", - " print(\"\\nSuccessfully extracted data.\")\n", + " print(\"\\nSuccessfully extracted proximate data.\")\n", " display(df.head())\n", " else:\n", - " print(\"\\nExtraction returned no data. Check the logs above for errors.\")\n", + " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", + "\n", + "if 'ultimate' in locals():\n", + " try:\n", + " df2 = ultimate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted Ultimate data.\")\n", + " display(df2.head())\n", + " else:\n", + " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", " except Exception as e:\n", - " print(f\"\\nAn error occurred during extraction: {e}\")" + " print(f\"\\nAn error occurred during extraction: {e}\")\n", + "\n", + "if 'cmpana' in locals():\n", + " try:\n", + " df3 = cmpana.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted CmpAna data.\")\n", + " display(df3.head())\n", + " else:\n", + " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", + " finally:\n", + " print(\"\\nCmp Ana extraction process completed.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "df.clean_names()" + "def clean_the_gsheets(df):\n", + " # 1. Clean names and drop rows\n", + " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + "\n", + " # 2. Coerce types (using errors='coerce' handles messy string data)\n", + " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", + " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Dates\n", + " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", + " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert remaining objects to best possible types (like strings)\n", + " df = df.convert_dtypes()\n", + " \n", + " return df # Return the FULL dataframe, not just .head()" ] }, { @@ -69,7 +1624,11 @@ "metadata": {}, "outputs": [], "source": [ - "df[['Record_ID', 'Source_codename', 'Prepared_sample']]" + "dataframes = [df, df2, df3]\n", + "\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", + "\n", + "clean_dataframes[2].head()\n" ] }, { @@ -78,10 +1637,11 @@ "metadata": {}, "outputs": [], "source": [ - "## Convert 'Value' column to numeric\n", - "pv_df = df[['Parameter', 'Value']]\n", + "summary_stats = clean_dataframes[0].\\\n", + " groupby(['resource', 'parameter'])['value'].\\\n", + " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", "\n", - "pv_df['Value'] = pd.to_numeric(pv_df['Value'], errors='coerce')" + "summary_stats" ] }, { @@ -90,13 +1650,12 @@ "metadata": {}, "outputs": [], "source": [ - "pv_df\n", - "\n", - "if pv_df.groupby('Parameter').mean() is not None:\n", - " print(\"Mean values by Parameter:\")\n", - " display(pv_df.groupby('Parameter').mean())\n", - "else:\n", - " print(\"No numeric values available to compute means.\")" + "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", + " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", + " agg({'value': 'mean'}).\\\n", + " query('value > 30').\\\n", + " sort_values(by='value', ascending=False).\\\n", + " round({'value': 1})\n" ] }, { @@ -105,9 +1664,18 @@ "metadata": {}, "outputs": [], "source": [ - "df2 = df[['Resource', 'Parameter', 'Value', 'Unit']]\n", + "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", + "\n", + "def is_it_volatile_solids(df):\n", + " df['check'] = \"VS\"\n", + "\n", + " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", + " return df\n", "\n", - "df2['Value'] = pd.to_numeric(df2['Value'], errors='coerce')" + "is_it_volatile_solids(df)\n", + "\n", + "df[['check', 'parameter']]\n", + "\n" ] }, { @@ -116,7 +1684,17 @@ "metadata": {}, "outputs": [], "source": [ - "df2" + "\n", + "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", + "def sqrtvalue(df):\n", + " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", + " return df\n", + "\n", + "#List comprehension to apply sqrtvalue to each DataFrame\n", + "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", + "\n", + "# Display the head of the third DataFrame\n", + "clean_rooted_df[2].head()" ] }, { @@ -125,25 +1703,2462 @@ "metadata": {}, "outputs": [], "source": [ - "df2 = df2.dropna(subset=['Value'])" + "cmpana_raw = cmpana.extract(project_root=project_root)\n", + "\n", + "cmpana_raw.head()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connected to database.\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "0dabe9e2-728a-486d-9dd1-c1edd094e9c2", + "rows": [ + [ + "0", + "1", + "Tomatoes for processing", + null, + null, + null + ], + [ + "1", + "2", + "Grapes", + null, + null, + null + ], + [ + "2", + "3", + "Almonds", + null, + null, + null + ], + [ + "3", + "4", + "Walnuts", + null, + null, + null + ], + [ + "4", + "5", + "Sweet potatoes", + null, + null, + null + ], + [ + "5", + "6", + "Algae", + null, + null, + null + ], + [ + "6", + "7", + "Olives - processing", + null, + null, + null + ], + [ + "7", + "8", + "Corn - all", + null, + null, + null + ], + [ + "8", + "9", + "Hay - alfalfa", + null, + null, + null + ], + [ + "9", + "10", + "Silage - wheat", + null, + null, + null + ], + [ + "10", + "11", + "Rice", + null, + null, + null + ], + [ + "11", + "12", + "Peaches", + null, + null, + null + ], + [ + "12", + "13", + "Potatoes", + null, + null, + null + ], + [ + "13", + "14", + "Cucumbers and pickles", + null, + null, + null + ], + [ + "14", + "15", + "Pistachios", + null, + null, + null + ], + [ + "15", + "16", + "Cotton", + null, + null, + null + ], + [ + "16", + "17", + "Olives - market", + null, + null, + null + ], + [ + "17", + "18", + "", + null, + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 18 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idnamenotedescriptionuri
01Tomatoes for processingNoneNoneNone
12GrapesNoneNoneNone
23AlmondsNoneNoneNone
34WalnutsNoneNoneNone
45Sweet potatoesNoneNoneNone
56AlgaeNoneNoneNone
67Olives - processingNoneNoneNone
78Corn - allNoneNoneNone
89Hay - alfalfaNoneNoneNone
910Silage - wheatNoneNoneNone
1011RiceNoneNoneNone
1112PeachesNoneNoneNone
1213PotatoesNoneNoneNone
1314Cucumbers and picklesNoneNoneNone
1415PistachiosNoneNoneNone
1516CottonNoneNoneNone
1617Olives - marketNoneNoneNone
1718NoneNoneNone
\n", + "
" + ], + "text/plain": [ + " id name note description uri\n", + "0 1 Tomatoes for processing None None None\n", + "1 2 Grapes None None None\n", + "2 3 Almonds None None None\n", + "3 4 Walnuts None None None\n", + "4 5 Sweet potatoes None None None\n", + "5 6 Algae None None None\n", + "6 7 Olives - processing None None None\n", + "7 8 Corn - all None None None\n", + "8 9 Hay - alfalfa None None None\n", + "9 10 Silage - wheat None None None\n", + "10 11 Rice None None None\n", + "11 12 Peaches None None None\n", + "12 13 Potatoes None None None\n", + "13 14 Cucumbers and pickles None None None\n", + "14 15 Pistachios None None None\n", + "15 16 Cotton None None None\n", + "16 17 Olives - market None None None\n", + "17 18 None None None" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "summary_stats = df2.groupby(['Resource', 'Parameter'])['Value'].agg(['mean', 'median', 'min', 'max', 'std', 'count'])" + "from sqlmodel import Session, select, create_engine\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "\n", + "# Database Connection\n", + "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", + "engine = create_engine(DATABASE_URL)\n", + "print(f\"Connected to database.\")\n", + "\n", + "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", + "\n", + "#reorders columns so id and name are first\n", + "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", + "\n", + "primary_ag_product = primary_ag_product[[*cols]]\n", + "\n", + "primary_ag_product\n" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "52090a35-9379-4dc9-b918-583b23513ddd", + "rows": [ + [ + "0", + null, + "1", + "Tomatoes for processing", + null, + null + ], + [ + "1", + null, + "2", + "Grapes", + null, + null + ], + [ + "2", + null, + "3", + "Almonds", + null, + null + ], + [ + "3", + null, + "4", + "Walnuts", + null, + null + ], + [ + "4", + null, + "5", + "Sweet potatoes", + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
descriptionidnamenoteuri
0None1Tomatoes for processingNoneNone
1None2GrapesNoneNone
2None3AlmondsNoneNone
3None4WalnutsNoneNone
4None5Sweet potatoesNoneNone
\n", + "
" + ], + "text/plain": [ + " description id name note uri\n", + "0 None 1 Tomatoes for processing None None\n", + "1 None 2 Grapes None None\n", + "2 None 3 Almonds None None\n", + "3 None 4 Walnuts None None\n", + "4 None 5 Sweet potatoes None None" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "# --- project root discovery (unchanged) ---\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# --- imports ---\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", + "\n", + "# --- query + dataframe ---\n", + "with Session(engine) as db:\n", + " stmt = select(*PrimaryAgProduct.__table__.columns)\n", + " rows = db.execute(stmt).mappings().all()\n", + "\n", + "df = pd.DataFrame(rows)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
21:07:54.222 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "21:07:54.222 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
21:07:59.598 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "21:07:59.598 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
21:07:59.601 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "21:07:59.601 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", + " replace_name_with_id_df,\n", + ") \n", + "\n", + "#This extractst the raw proximate data\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", + "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", + "\n", + "#this replaces the names with IDs\n", + "with Session(engine) as db:\n", + " parameter_ids = replace_name_with_id_df(\n", + " db=db,\n", + " df=test_df,\n", + " ref_model=Parameter,\n", + " name_column_name=\"name\", # column in df + table\n", + " id_column_name=\"id\", # PK column in table\n", + " final_column_name=\"parameter_id\"\n", + " )\n", + "\n", + "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", + "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", + "\n", + "#resource_id_mapping\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "string", + "type": "string" + }, + { + "name": "record_id", + "rawType": "string", + "type": "string" + }, + { + "name": "prepared_sample", + "rawType": "string", + "type": "string" + }, + { + "name": "resource", + "rawType": "string", + "type": "string" + }, + { + "name": "preparation_method", + "rawType": "string", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "string", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "string", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "Int32", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "string", + "type": "string" + }, + { + "name": "value", + "rawType": "Float32", + "type": "float" + }, + { + "name": "unit", + "rawType": "string", + "type": "string" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "string", + "type": "string" + }, + { + "name": "note", + "rawType": "string", + "type": "string" + }, + { + "name": "analysis_type", + "rawType": "string", + "type": "string" + }, + { + "name": "equipment", + "rawType": "string", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "string", + "type": "string" + }, + { + "name": "analyst_email", + "rawType": "string", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "string", + "type": "string" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "91fad99c-9137-48ce-af0f-4004b3241e88", + "rows": [ + [ + "0", + "3EE2993D-86E3-1F16-C7EA-F8D555E114", + "(85)E114", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "14.16", + "% dry weight", + "2025-01-23 09:00:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "1", + "46878EF9-1226-22A0-D5D8-CF65E241CB", + "(85)41CB", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "14.18", + "% dry weight", + "2025-01-23 09:00:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "2", + "76A7A2F4-C4E4-E60F-1187-DEC6E02246", + "(85)2246", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "14.12", + "% dry weight", + "2025-01-23 09:00:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "3", + "7A136832-286B-07CB-62DE-ACF52F9311", + "(85)9311", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "15.74", + "% dry weight", + "2025-01-23 09:00:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "4", + "B709ECEE-F9A6-A55D-A59E-93B7B863D7", + "(85)63D7", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "15.75", + "% dry weight", + "2025-01-23 09:01:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "5", + "35080403-BB4B-776B-E045-4627C307BD", + "(85)07BD", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "15.68", + "% dry weight", + "2025-01-23 09:01:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "6", + "D83F3328-931D-582E-B563-4DFF118C05", + "(85)8C05", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "9.88", + "% dry weight", + "2025-01-23 09:01:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "7", + "B3F6AA18-E51A-2585-8417-113DE0F026", + "(85)F026", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "9.25", + "% dry weight", + "2025-01-23 09:01:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "8", + "094CFB14-0D19-C54E-51C4-6BA548A6D4", + "(85)A6D4", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "9.43", + "% dry weight", + "2025-01-23 09:02:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "9", + "99D17377-62B8-2124-5391-62BC1F4630", + "(85)4630", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "11.22", + "% dry weight", + "2025-01-23 09:02:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "10", + "9B784971-B92C-A870-DA69-C4D771B716", + "(85)B716", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "10.51", + "% dry weight", + "2025-01-23 09:02:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "11", + "57C508A2-7489-2AFB-23B2-1630C38BD5", + "(85)8BD5", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "10.72", + "% dry weight", + "2025-01-23 09:02:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "12", + "40EAE6DB-4CD6-3574-949F-70B03A51C8", + "(85)51C8", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(85)1", + "39.54", + "% dry weight", + "2025-01-23 09:03:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "13", + "A752F059-ECF4-FA5B-A9D2-7DF8963329", + "(85)3329", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(85)2", + "39.9", + "% dry weight", + "2025-01-23 09:03:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "14", + "BEB4369F-D40F-DF6A-30F8-75B91C5B1B", + "(85)5B1B", + "Oak-TmPm01O(85)", + "Tomato pomace", + "Oven dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(85)3", + "40.88", + "% dry weight", + "2025-01-23 09:03:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "15", + "00CAC475-8E18-2318-AA6A-2AE068CADF", + "(D2)CADF", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "19.04", + "% dry weight", + "2025-01-23 09:03:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "16", + "2DA01B52-D301-53C9-220D-A51216BDB6", + "(D2)BDB6", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "18.9", + "% dry weight", + "2025-01-23 09:04:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "17", + "F8D1C324-F3F5-7754-494E-7BBFB262D0", + "(D2)62D0", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "18.98", + "% dry weight", + "2025-01-23 09:04:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "18", + "84FE8C5C-5D37-B0EA-6488-B883CEA7F3", + "(D2)A7F3", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "21.15", + "% dry weight", + "2025-01-23 09:04:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "19", + "E705EAF1-69EB-ADB3-620B-1DD93F4D07", + "(D2)4D07", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "21.0", + "% dry weight", + "2025-01-23 09:04:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "20", + "F6A97D04-E898-06F1-A90B-0EE254D758", + "(D2)D758", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "21.08", + "% dry weight", + "2025-01-23 09:05:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "21", + "6948E00A-4858-6F19-23BC-AA016F12CB", + "(D2)12CB", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "8.84", + "% dry weight", + "2025-01-23 09:05:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "22", + "EEC9EA54-3C1A-6154-B953-401C89CEA7", + "(D2)CEA7", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "8.46", + "% dry weight", + "2025-01-23 09:05:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "23", + "62E4A908-ED27-590F-0DD2-EA254F1E4C", + "(D2)1E4C", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "8.72", + "% dry weight", + "2025-01-23 09:05:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "24", + "AA2BBE0D-FD98-98F9-9880-CDB56142A2", + "(D2)42A2", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "10.04", + "% dry weight", + "2025-01-23 09:06:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "25", + "5186F84F-8BBB-98A7-8FF9-CF85CCD344", + "(D2)D344", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "9.62", + "% dry weight", + "2025-01-23 09:06:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "26", + "2BAF4360-89A7-C605-2141-827901DA56", + "(D2)DA56", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "9.91", + "% dry weight", + "2025-01-23 09:06:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "27", + "B9D049D7-363C-365F-3750-331CEC2FAE", + "(D2)2FAE", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(D2)1", + "30.53", + "% dry weight", + "2025-01-23 09:06:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "28", + "4E056928-76F1-99E1-4F78-FD31CC0B71", + "(D2)0B71", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(D2)2", + "31.57", + "% dry weight", + "2025-01-23 09:07:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "29", + "4734CD60-9A40-8ECD-7E0D-FA2AD6D96D", + "(D2)D96D", + "Pin-TmPm02O(D2)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(D2)3", + "31.95", + "% dry weight", + "2025-01-23 09:07:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "30", + "2CD57693-9038-EC9E-EC61-E91793A602", + "(28)A602", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "11.71", + "% dry weight", + "2025-01-23 09:07:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "31", + "BD14399E-2A3B-56A9-AC9C-1689DA75A7", + "(28)75A7", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "13.66", + "% dry weight", + "2025-01-23 09:07:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "32", + "6635AA49-3247-3A08-D00F-CA5447491E", + "(28)491E", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "13.84", + "% dry weight", + "2025-01-23 09:08:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "33", + "EE7D6738-BEE5-87A1-6387-89AE03CFC4", + "(28)CFC4", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "13.01", + "% dry weight", + "2025-01-23 09:08:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "34", + "ACD2BFD6-EA7C-8CBB-F4DA-CAA860F6C9", + "(28)F6C9", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "15.17", + "% dry weight", + "2025-01-23 09:08:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "35", + "1BFB37F6-A5D7-E88E-84A4-DBA38059B5", + "(28)59B5", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "15.37", + "% dry weight", + "2025-01-23 09:08:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "36", + "D6CD55E5-19B7-5E3A-EEE6-95564F8C33", + "(28)8C33", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "9.36", + "% dry weight", + "2025-01-23 09:09:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "37", + "17DE0AED-2CC5-2B5A-E521-049F78F0FE", + "(28)F0FE", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "11.26", + "% dry weight", + "2025-01-23 09:09:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "38", + "4D593E32-1FDA-373B-EE45-38FB01847B", + "(28)847B", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "11.06", + "% dry weight", + "2025-01-23 09:09:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "2" + ], + [ + "39", + "84904964-58B9-A45E-3F79-6747BF7919", + "(28)7919", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "10.64", + "% dry weight", + "2025-01-23 09:09:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "40", + "808F6FAC-F9A6-A9EB-56D5-EECA3BA3D2", + "(28)A3D2", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "12.8", + "% dry weight", + "2025-01-23 09:10:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "41", + "67FD9F7C-8CE8-04F4-2CB5-6C42532429", + "(28)2429", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "12.56", + "% dry weight", + "2025-01-23 09:10:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "5" + ], + [ + "42", + "90E8067F-045F-3E6E-5BAA-94BEE9D0CF", + "(28)D0CF", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(28)1", + "40.12", + "% dry weight", + "2025-01-23 09:10:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "43", + "112546A5-714C-B29F-808B-CBDE6DE48E", + "(28)E48E", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(28)2", + "39.76", + "% dry weight", + "2025-01-23 09:10:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "44", + "2FF25721-ED5B-05E4-2DC5-5799261266", + "(28)1266", + "Riv-TmPm03O(28)", + "Tomato pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(28)3", + "40.34", + "% dry weight", + "2025-01-23 09:11:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "1" + ], + [ + "45", + "7BC7134E-F8BB-F153-DC50-67A1161509", + "(AB)1509", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(AB)1", + "7.95", + "% dry weight", + "2025-01-22 09:00:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "46", + "F442FF50-46CF-317D-9708-14F6461273", + "(AB)1273", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(AB)2", + "7.77", + "% dry weight", + "2025-01-22 09:00:16", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "47", + "C80175E1-8D21-920C-B982-A246941FCB", + "(AB)1FCB", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "3", + "Cmp04xk(AB)3", + "7.96", + "% dry weight", + "2025-01-22 09:00:31", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "6" + ], + [ + "48", + "63CF426B-99C5-3896-C472-49BBAE19CD", + "(AB)19CD", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "1", + "Cmp04xk(AB)1", + "8.84", + "% dry weight", + "2025-01-22 09:00:46", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ], + [ + "49", + "A9D902B0-033D-29B4-C477-1A7D3E57F4", + "(AB)57F4", + "Map-GpPm04O(AB)", + "Grape pomace", + "Oven Dry", + "RT vacuum sealed", + "Cmp04xk", + "2", + "Cmp04xk(AB)2", + "8.64", + "% dry weight", + "2025-01-22 09:01:01", + null, + "pass", + "", + "Chemical composition", + "", + "", + "xkang2@lbl.gov", + "ready", + "4" + ] + ], + "shape": { + "columns": 21, + "rows": 390 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cmp_uuid_033record_idprepared_sampleresourcepreparation_methodstorage_condexper_abbrevrepl_norepl_idvalue...created_atupdated_atqc_resultnoteanalysis_typeequipmentraw_data_urlanalyst_emailupload_statusparameter_id
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)114.16...2025-01-23 09:00:01NaTpassChemical compositionxkang2@lbl.govready6
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)214.18...2025-01-23 09:00:16NaTpassChemical compositionxkang2@lbl.govready6
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)314.12...2025-01-23 09:00:31NaTpassChemical compositionxkang2@lbl.govready6
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)115.74...2025-01-23 09:00:46NaTpassChemical compositionxkang2@lbl.govready4
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)215.75...2025-01-23 09:01:01NaTpassChemical compositionxkang2@lbl.govready4
..................................................................
38579799601-0E25-3832-3BBF-F4BD9EE973(A0)E973Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)217.290001...2025-03-04 09:55:00NaTpassChemical compositionxkang2@lbl.govnot ready5
386A4E3393F-72C3-A0ED-C041-105A33031A(A0)031AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)317.5...2025-03-04 09:55:15NaTpassChemical compositionxkang2@lbl.govnot ready5
3873D5A85F8-31A2-0CC2-74E6-05D8197C3A(A0)7C3AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1Cmp17xk(A0)135.970001...2025-03-04 09:55:30NaTpassChemical compositionxkang2@lbl.govnot ready3
388C7AE456E-4DCD-AE34-A0D0-BD39D04E42(A0)4E42Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)235.279999...2025-03-04 09:55:45NaTpassChemical compositionxkang2@lbl.govnot ready3
389A5E617A2-9831-3E9F-7072-6C5FB8129B(A0)129BHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)334.990002...2025-03-04 09:56:00NaTpassChemical compositionxkang2@lbl.govnot ready3
\n", + "

390 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " cmp_uuid_033 record_id prepared_sample \\\n", + "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", + "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", + "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", + "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", + "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", + ".. ... ... ... \n", + "385 79799601-0E25-3832-3BBF-F4BD9EE973 (A0)E973 Hum-AlmBr024KM2(A0) \n", + "386 A4E3393F-72C3-A0ED-C041-105A33031A (A0)031A Hum-AlmBr024KM2(A0) \n", + "387 3D5A85F8-31A2-0CC2-74E6-05D8197C3A (A0)7C3A Hum-AlmBr024KM2(A0) \n", + "388 C7AE456E-4DCD-AE34-A0D0-BD39D04E42 (A0)4E42 Hum-AlmBr024KM2(A0) \n", + "389 A5E617A2-9831-3E9F-7072-6C5FB8129B (A0)129B Hum-AlmBr024KM2(A0) \n", + "\n", + " resource preparation_method storage_cond exper_abbrev \\\n", + "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", + ".. ... ... ... ... \n", + "385 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "386 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "387 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "388 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "389 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", + "\n", + " repl_no repl_id value ... created_at updated_at \\\n", + "0 1 Cmp04xk(85)1 14.16 ... 2025-01-23 09:00:01 NaT \n", + "1 2 Cmp04xk(85)2 14.18 ... 2025-01-23 09:00:16 NaT \n", + "2 3 Cmp04xk(85)3 14.12 ... 2025-01-23 09:00:31 NaT \n", + "3 1 Cmp04xk(85)1 15.74 ... 2025-01-23 09:00:46 NaT \n", + "4 2 Cmp04xk(85)2 15.75 ... 2025-01-23 09:01:01 NaT \n", + ".. ... ... ... ... ... ... \n", + "385 2 Cmp17xk(A0)2 17.290001 ... 2025-03-04 09:55:00 NaT \n", + "386 3 Cmp17xk(A0)3 17.5 ... 2025-03-04 09:55:15 NaT \n", + "387 1 Cmp17xk(A0)1 35.970001 ... 2025-03-04 09:55:30 NaT \n", + "388 2 Cmp17xk(A0)2 35.279999 ... 2025-03-04 09:55:45 NaT \n", + "389 3 Cmp17xk(A0)3 34.990002 ... 2025-03-04 09:56:00 NaT \n", + "\n", + " qc_result note analysis_type equipment raw_data_url \\\n", + "0 pass Chemical composition \n", + "1 pass Chemical composition \n", + "2 pass Chemical composition \n", + "3 pass Chemical composition \n", + "4 pass Chemical composition \n", + ".. ... ... ... ... ... \n", + "385 pass Chemical composition \n", + "386 pass Chemical composition \n", + "387 pass Chemical composition \n", + "388 pass Chemical composition \n", + "389 pass Chemical composition \n", + "\n", + " analyst_email upload_status parameter_id \n", + "0 xkang2@lbl.gov ready 6 \n", + "1 xkang2@lbl.gov ready 6 \n", + "2 xkang2@lbl.gov ready 6 \n", + "3 xkang2@lbl.gov ready 4 \n", + "4 xkang2@lbl.gov ready 4 \n", + ".. ... ... ... \n", + "385 xkang2@lbl.gov not ready 5 \n", + "386 xkang2@lbl.gov not ready 5 \n", + "387 xkang2@lbl.gov not ready 3 \n", + "388 xkang2@lbl.gov not ready 3 \n", + "389 xkang2@lbl.gov not ready 3 \n", + "\n", + "[390 rows x 21 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parameter_ids\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "summary_stats" + "import numpy as np\n", + "from prefect import get_run_logger\n", + "\n", + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "resource['name'] = resource['name'].str.lower()\n", + "\n", + "resource['name'] = resource['name'].replace('', np.nan)\n", + "resource.dropna(subset=['name'], inplace=True)\n", + "\n", + "\n", + "resource\n" ] }, { @@ -152,7 +4167,12 @@ "metadata": {}, "outputs": [], "source": [ - "\n" + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "#this converts the entire dataframe to lowercase\n", + "df = df.map(lambda x: x.lower() if isinstance(x, str) else x)\n", + "\n", + "print(df)" ] }, { @@ -160,18 +4180,32 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "field_sample = pd.read_sql_query(\"SELECT * FROM field_sample\", con=engine)\n", + "\n", + "field_sample = field_sample[['id', 'name', 'resource_id']]\n", + "\n", + "field_sample" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "default", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.12" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py new file mode 100644 index 00000000..6c92cb4f --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -0,0 +1,81 @@ +from typing import Type, TypeVar, Any + +import pandas as pd +from sqlalchemy.orm import Session +from sqlalchemy import select + +ModelType = TypeVar("ModelType", bound=Any) + +def replace_id_with_name_df( + db: Session, + df: pd.DataFrame, + ref_model: Type[ModelType], + id_column_name: str, + name_column_name: str, +) -> pd.DataFrame: + # Fetch reference table rows as mappings + rows = db.execute( + select(*ref_model.__table__.columns) + ).mappings().all() + + # Build ID → name map + id_to_name_map = { + row[id_column_name]: row[name_column_name] + for row in rows + } + + df_copy = df.copy() + df_copy[name_column_name] = df_copy[id_column_name].map(id_to_name_map) + df_copy = df_copy.drop(columns=[id_column_name]) + + return df_copy + + +def replace_name_with_id_df( + db: Session, + df: pd.DataFrame, + ref_model: Type[ModelType], + name_column_name: str, + id_column_name: str, + final_column_name: str +) -> pd.DataFrame: + # Fetch reference table rows + rows = db.execute( + select(*ref_model.__table__.columns) + ).mappings().all() + + name_to_id_map = { + row[name_column_name]: row[id_column_name] + for row in rows + } + + df_copy = df.copy() + + unique_names = set(df_copy[name_column_name].dropna().unique()) + new_names = unique_names - set(name_to_id_map.keys()) + + if new_names: + for name in new_names: + new_record = ref_model(**{name_column_name: name}) + db.add(new_record) + + # Commit once, not per row + db.commit() + + # Refresh and update lookup + refreshed = db.execute( + select(ref_model).where( + getattr(ref_model, name_column_name).in_(new_names) + ) + ).scalars().all() + + for record in refreshed: + name_to_id_map[getattr(record, name_column_name)] = getattr( + record, id_column_name + ) + + df_copy[id_column_name] = df_copy[name_column_name].map(name_to_id_map) + df_copy = df_copy.drop(columns=[name_column_name]) + df_copy = df_copy.rename(columns={id_column_name: final_column_name}) + + return df_copy diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql deleted file mode 100644 index c0d3337d..00000000 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/seed_biomass.sql +++ /dev/null @@ -1,15 +0,0 @@ --- This script inserts 10 sample rows into the biomass table. --- Note: This assumes that valid records exist in the primary_product, taxonomy, and biomass_type tables --- for the foreign key IDs used below. - -INSERT INTO biomass (biomass_name, primary_product_id, taxonomy_id, biomass_type_id, biomass_notes) VALUES -('Corn Stover', 1, 101, 1, 'Lignocellulosic biomass from corn stalks and leaves.'), -('Switchgrass', 2, 102, 1, 'Perennial grass used as a dedicated energy crop.'), -('Pine Wood Chips', 3, 103, 2, 'Woody biomass from pine trees, chipped for processing.'), -('Algae Culture', 4, 104, 3, 'Aquatic biomass grown for biofuel production.'), -('Municipal Solid Waste (MSW)', 5, 105, 4, 'Organic fraction of household waste.'), -('Food Waste', 6, 106, 4, 'Post-consumer food scraps from restaurants.'), -('Manure Slurry', 7, 107, 5, 'Animal waste from dairy farms, used in anaerobic digestion.'), -('Miscanthus Giganteus', 2, 108, 1, 'High-yield sterile hybrid grass for bioenergy.'), -('Willow Coppice', 3, 109, 2, 'Short-rotation coppice willow for biomass energy.'), -('Sugarcane Bagasse', 1, 110, 1, 'Fibrous residue left after sugarcane crushing.'); From 5e961aa19d87a1df708b83051e01a15efeb1fd8c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Sat, 3 Jan 2026 08:39:43 -0700 Subject: [PATCH 05/10] modifying name_id_swap_function --- .../pipeline/utils/name_id_swap.py | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py index 6c92cb4f..d9290968 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -31,51 +31,60 @@ def replace_id_with_name_df( return df_copy + def replace_name_with_id_df( db: Session, df: pd.DataFrame, - ref_model: Type[ModelType], - name_column_name: str, + ref_model, + df_name_column: str, + model_name_attr: str, id_column_name: str, - final_column_name: str + final_column_name: str, ) -> pd.DataFrame: - # Fetch reference table rows + """ + Replace a DataFrame name column with foreign key IDs from a SQLAlchemy table. + Creates missing reference records if needed. + """ + + # 1. Fetch existing reference rows (name + id only) rows = db.execute( - select(*ref_model.__table__.columns) - ).mappings().all() + select( + getattr(ref_model, model_name_attr), + getattr(ref_model, id_column_name), + ) + ).all() - name_to_id_map = { - row[name_column_name]: row[id_column_name] - for row in rows - } + name_to_id_map = {name: id_ for name, id_ in rows} df_copy = df.copy() - unique_names = set(df_copy[name_column_name].dropna().unique()) + # 2. Determine which names are new + unique_names = set(df_copy[df_name_column].dropna().unique()) new_names = unique_names - set(name_to_id_map.keys()) + # 3. Insert missing reference rows if new_names: for name in new_names: - new_record = ref_model(**{name_column_name: name}) + new_record = ref_model(**{model_name_attr: name}) db.add(new_record) - # Commit once, not per row - db.commit() + # Flush to get IDs without ending the transaction + db.flush() - # Refresh and update lookup + # Re-query just-created rows refreshed = db.execute( select(ref_model).where( - getattr(ref_model, name_column_name).in_(new_names) + getattr(ref_model, model_name_attr).in_(new_names) ) ).scalars().all() for record in refreshed: - name_to_id_map[getattr(record, name_column_name)] = getattr( - record, id_column_name - ) + name_to_id_map[ + getattr(record, model_name_attr) + ] = getattr(record, id_column_name) - df_copy[id_column_name] = df_copy[name_column_name].map(name_to_id_map) - df_copy = df_copy.drop(columns=[name_column_name]) - df_copy = df_copy.rename(columns={id_column_name: final_column_name}) + # 4. Replace name column with ID column + df_copy[final_column_name] = df_copy[df_name_column].map(name_to_id_map) + df_copy = df_copy.drop(columns=[df_name_column]) return df_copy From 6d7990b252f7f86b948946ce958cc48bbb830f2c Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Sun, 4 Jan 2026 20:06:00 -0700 Subject: [PATCH 06/10] was trying to fix dev container kernel issue. Did not succeed. Also made new ETL notebook and modified some of the gsheet extraction notebook --- pixi.lock | 1606 +++++- pixi.toml | 1 + .../pipeline/utils/etl_notebook.ipynb | 1787 ++++++ .../utils/gsheet_extraction_notebook.ipynb | 4929 ++++------------- .../pipeline/utils/name_id_swap.py | 8 +- 5 files changed, 4407 insertions(+), 3924 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb diff --git a/pixi.lock b/pixi.lock index faeb4843..5581bcfd 100644 --- a/pixi.lock +++ b/pixi.lock @@ -10,13 +10,25 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/argon2-cffi-bindings-25.1.0-py312h4c3975b_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/asyncpg-0.29.0-py312h66e93f0_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bcrypt-5.0.0-py312h868fb18_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.2.0-py312hdb49522_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py312h460c074_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -28,6 +40,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/cryptography-46.0.3-py312ha4b625e_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.18-py312h8285ef7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/docker-cli-28.3.1-h73b1eb8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/docker-compose-2.39.2-h280c20c_0.conda @@ -35,22 +48,46 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda @@ -67,12 +104,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.2-he9a06e4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.3-py312h8a5da7c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.4-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -81,7 +127,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.1.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda @@ -94,30 +142,49 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.12-hd63d673_1_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/pyyaml-6.0.3-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.1.0-py312hfb55c3c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.30.0-py312h868fb18_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.3-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py312hd9148b4_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/wrapt-2.0.1-py312h4c3975b_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h387f397_9.conda @@ -129,12 +196,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -147,7 +210,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -156,29 +218,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -195,7 +248,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -210,19 +262,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/14/9a39b7c9e007968411bc3c843cc14cf15437510c0a9991f080cab654fd16/regex-2025.10.23-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3e/cd/49ce51767b879cde77e7ad9fae164ea15dce3616fe591d9ea1df51152706/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/61/62/150c841f24cda9e30f588ef396ed83f64cfdc13b92d2f925bb96df337ba9/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -230,7 +277,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -249,13 +295,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/59/61d8e9f1734069049abe9e593961de602397c7194712346906c075fec65f/uv-0.9.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels @@ -264,13 +307,25 @@ environments: linux-aarch64: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/argon2-cffi-bindings-25.1.0-py312hcd1a082_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/asyncpg-0.29.0-py312hb2c0f52_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bcrypt-5.0.0-py312h5eb8f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-python-1.1.0-py312h1ab2c47_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py312h2fc7fbd_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -282,6 +337,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cryptography-46.0.3-py312h4cd2d69_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/debugpy-1.8.17-py312hf55c4e8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/docker-cli-28.3.1-h06eaf92_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/docker-compose-2.39.2-h80f16a2_0.conda @@ -289,22 +345,46 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-h9df1782_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda @@ -321,12 +401,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/markupsafe-3.0.3-py312hd077ced_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -335,7 +424,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/psutil-7.1.2-py312hd41f8a7_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda @@ -348,30 +439,49 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.12-hcfbf8c2_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pyyaml-6.0.3-py312ha4530ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pyzmq-27.1.0-py312h4552c38_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rpds-py-0.30.0-py312h75d7d99_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tornado-6.5.2-py312hefbd42c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ukkonen-1.0.1-py312h451a7dd_5.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wrapt-1.17.3-py312hcd1a082_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zeromq-4.3.5-hefbcea8_9.conda @@ -383,12 +493,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -401,7 +507,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -410,29 +515,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -449,7 +545,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -464,19 +559,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/95/888f069c89e7729732a6d7cca37f76b44bfb53a1e35dda8a2c7b65c1b992/regex-2025.10.23-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/ea/42d243d3a586beb72c77fa5def0487daf827210069a95f36328e869599ea/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/c5/346c7094344a60419764b4b1334d9e0285031c961176ff88ffb652405b0c/ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux2014_aarch64.whl @@ -484,7 +574,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -503,13 +592,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/ac/090dbde63abb56001190392d29ca2aa654eebc146a693b5dda68da0df2fb/uv-0.9.7-py3-none-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels @@ -517,14 +603,26 @@ environments: - pypi: ./src/ca_biositing/webservice osx-64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/argon2-cffi-bindings-25.1.0-py313hf050af9_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/asyncpg-0.29.0-py313ha37c0e0_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/bcrypt-5.0.0-py313hcc225dc_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/brotli-python-1.2.0-py313hd4eab94_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cffi-2.0.0-py313hf57695f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -536,6 +634,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/cryptography-46.0.3-py313h0218d6d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/debugpy-1.8.17-py313hff8d55d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/docker-cli-28.3.1-h2287256_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/docker-compose-2.39.2-h5282f48_0.conda @@ -543,21 +642,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.21.3-h37d8d59_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.4-h3d58e20_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libedit-3.1.20250104-pl5321ha958ccf_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.1-h21dd04a_0.conda @@ -567,12 +690,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libsodium-1.0.20-hfdf4475_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.50.4-h39a8b3b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/markupsafe-3.0.3-py313h0f4d31d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.5.4-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -581,42 +713,65 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/psutil-7.1.3-py313hcb05632_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pynacl-1.6.0-py313hf050af9_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-core-12.1-py313h07bcf3a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-framework-cocoa-12.1-py313hf669bc3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-8.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.0.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.13.9-h17c18a5_101_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.9-h4df99d1_101.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py313h0f4d31d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyzmq-27.1.0-py312hb7d603e_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h7cca4af_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/rpds-py-0.30.0-py313hcc225dc_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tornado-6.5.2-py313h585f44e_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ukkonen-1.0.1-py313hc551f4f_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/wrapt-2.0.0-py313hf050af9_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zeromq-4.3.5-h6c33b1e_9.conda @@ -628,12 +783,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -646,7 +797,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -655,29 +805,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -693,7 +834,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -708,19 +848,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4c/93/181070cd1aa2fa541ff2d3afcf763ceecd4937b34c615fa92765020a6c90/regex-2025.10.23-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d3/03/ce566d92611dfac0085c2f4b048cd53ed7c274a5c05974b882a908d540a2/rpds_py-0.28.0-cp313-cp313-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d7/ae/e3811f05415594025e96000349d3400978adaed88d8f98d494352d9761ee/ruamel.yaml.clib-0.2.14-cp313-cp313-macosx_10_13_universal2.whl @@ -728,7 +863,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -747,13 +881,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6f/b7/1b1ff8dfde05e9d27abf29ebf22da48428fe1e16f0b4d65a839bd2211303/uv-0.9.7-py3-none-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b4/01/543ce9ecaa8dd288b5bd830db221a1508b3662196022f001818f4de36ef3/whenever-0.9.3-cp313-cp313-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl @@ -762,14 +893,26 @@ environments: - pypi: ./src/ca_biositing/webservice osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/argon2-cffi-bindings-25.1.0-py313h6535dbc_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/asyncpg-0.29.0-py313h20a7fcf_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bcrypt-5.0.0-py313h2c089d5_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.2.0-py313hde1f3bb_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.10.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cffi-2.0.0-py313h224173a_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda @@ -781,6 +924,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-46.0.3-py313h76c770c_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.19-py313hc37fe24_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/docker-cli-28.3.1-hbc156a2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/docker-compose-2.39.2-h925e9cb_0.conda @@ -788,21 +932,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.6.0-pyhfa0c392_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.21.3-h237132a_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321hafb1f1b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda @@ -812,12 +980,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsodium-1.0.20-h99b78c6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py313h7d74516_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda @@ -826,42 +1003,65 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.3.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.1.3-py313h9734d34_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pynacl-1.6.1-py313h6535dbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-core-12.1-py313h40b429f_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-framework-cocoa-12.1-py313hcc5defa_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-8.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.0.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.11-hfc2f54d_100_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/pyyaml-6.0.3-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-27.1.0-py312hd65ceae_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/rpds-py-0.30.0-py313h2c089d5_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tornado-6.5.4-py313h6535dbc_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ukkonen-1.0.1-py313hc50a443_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.0.1-py313h6535dbc_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h888dc83_9.conda @@ -873,12 +1073,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/f1/318762320d966e528dfb9e6be5953fe7df2952156f15ba857cbccafb630c/apprise-1.9.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz - pypi: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl @@ -891,7 +1087,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -900,29 +1095,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -938,7 +1124,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/e6/96494debc7f5fb9730a75abfa600efaeac7fa40683ba52b478f0df294ee8/prefect-3.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl @@ -953,19 +1138,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/c5/9d37fbe3a40ed8dda78c23e1263002497540c0d1522ed75482ef6c2000f0/regex-2025.10.23-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/00/34/1c61da1b25592b86fd285bd7bd8422f4c9d748a7373b46126f9ae792a004/rpds_py-0.28.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d6/03/a1baa5b94f71383913f21b96172fb3a2eb5576a4637729adbf7cd9f797f8/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_11_0_arm64.whl @@ -973,7 +1153,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl @@ -992,13 +1171,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f5/7d/b618174d8a8216af350398ace03805b2b2df6267b1745abf45556c2fda58/uv-0.9.7-py3-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7b/d7/d89a6f1b1ea4a2768c813443d7be007377402fa0db99c62c393965165159/whenever-0.9.3-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c3/78/4d6d68555a92cb97b4c192759c4ab585c5cb23490f64d4ddf12c66a3b051/xarray-2025.10.1-py3-none-any.whl @@ -1007,14 +1183,26 @@ environments: - pypi: ./src/ca_biositing/webservice win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/argon2-cffi-bindings-25.1.0-py313h5ea7bf4_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/asyncpg-0.29.0-py313ha7868ed_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.4.0-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/backports.zstd-1.2.0-py313h2a31948_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bcrypt-5.0.0-py313hfbe8231_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/brotli-python-1.2.0-py313h3ebfc14_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025.11.12-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.5.0-pyhd8ed1ab_0.conda @@ -1026,6 +1214,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/cryptography-46.0.3-py313hf5c5e30_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.19-py313h927ade5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/docker-cli-28.3.1-h6a83c73_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/docker-compose-2.40.1-h6a83c73_0.conda @@ -1033,20 +1222,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.20.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/identify-2.6.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/invoke-2.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-7.1.0-pyh6dadd2b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.8.0-pyhe2676ad_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.7.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyh6dadd2b_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda @@ -1054,18 +1268,29 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libsodium-1.0.20-hc70643c_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/markupsafe-3.0.3-py313hd650c13_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pip-25.3-pyh145f28c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhf9edf01_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.5.1-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/psutil-7.1.3-py313h5fd188c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda @@ -1077,49 +1302,67 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.13.11-h09917c8_100_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pywin32-311-py313h40c08fc_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh07e9846_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/win-64/pywinpty-2.0.15-py313h5813708_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py313hd650c13_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyzmq-27.1.0-py312hbb5da91_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/rpds-py-0.30.0-py313hfbe8231_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh6dadd2b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh6dadd2b_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.3-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tornado-6.5.4-py313h5ea7bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ukkonen-1.0.1-py313hf069bd2_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.6.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.35.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/win_inet_pton-1.1.0-pyh7428d3b_8.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/winpty-0.4.3-4.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/wrapt-2.0.1-py313h5ea7bf4_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zeromq-4.3.5-h5bddc39_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.23.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ef/39/b2181148075272edfbbd6d87e6cd78cc71dca243446fa3b381fd4116950b/aiosqlite-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz - - pypi: https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/df/343d125241f8cd3c9af58fd09688cf2bf59cc1edfd609adafef3556ce8ec/apprise-1.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/fc/1d7b80d0eb7b714984ce40efc78859c022cd930e402f599d8ca9e39c78a4/cachetools-6.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz @@ -1134,7 +1377,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/78/a850fed8aeef96d4a99043c90b818b2ed5419cd5b24a4049fd7cfb9f1471/fakeredis-2.33.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b6/2b/15ac29e524bf0dc8fcf8e90364c05709b3d4fa810a86421c6924ea18cccd/fastapi-0.114.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/07/a54c100da461ffc5968457823fcc665a48fb4b875c68bcfecbfe24a10dbe/google_auth_oauthlib-1.2.3-py3-none-any.whl @@ -1143,31 +1385,21 @@ environments: - pypi: https://files.pythonhosted.org/packages/9c/83/3b1d03d36f224edded98e9affd0467630fc09d766c0e56fb1498cbb04a9b/griffe-1.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c9/2f/ffb71b19208cf3d3e1e1323716de07bdb46e24c850f2c5627cb8157d9687/gspread_dataframe-4.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/7b/bca5613a0c3b542420cf92bd5e5fb8ebd5435ce1011a091f66bb7693285e/humanize-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/ff/b965a845fc0a9ca2331d8d6043e262cc834f5328ac0f1970db1622bac2cf/linkml-1.9.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/28/cdcbe1f0521a98b891dd30249513eef1ddcc7bb406be953b4a8d7825e68f/linkml_runtime-1.9.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/b4/55e885834c847ea610e111d87b9ed4768f0afdaeebc00cd46810f25029f6/lupa-2.6-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/c0/00c9809d8b9346eb238a6bbd5f83e846a4ce4503da94a4c08cb7284c325b/multipledispatch-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl @@ -1186,7 +1418,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/59/3d/68243e3e219d765a3012a1e396b3ea8566c3ea2b60283e27327c0f4415e9/prefect-3.6.7-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/99/10/72f6f213b8f0bce36eff21fda0a13271834e9eeff7f9609b01afdc253c79/py_key_value_aio-0.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/e4/b8b0a03ece72f47dce2307d36e1c34725b7223d209fc679315ffe6a4e2c3/py_key_value_shared-0.3.0-py3-none-any.whl @@ -1203,22 +1434,16 @@ environments: - pypi: https://files.pythonhosted.org/packages/9b/48/efb1b1d3f3aee8cfc9f256738ca6e79ec362edbfc3a3abecbaf84db04643/PyShEx-0.8.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/7d/ff5000e0882f2b3995bef20b667945d3faa9289b556295e4cc5d2e91f104/PyShExC-0.9.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz - - pypi: https://files.pythonhosted.org/packages/51/e5/fecf13f06e5e5f67e8837d777d1bc43fac0ed2b77a676804df5c34744727/python_json_logger-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/20/35d2baebacf357b562bd081936b66cd845775442973cb033a377fd639a84/rdflib-7.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/f0/8956f8a86b20d7bb9d6ac0187cf4cd54d8065bc9a1a09eb8011d4d326596/redis-7.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8b/00/6e29bb314e271a743170e53649db0fdb8e8ff0b64b4f425f5602f4eb9014/regex-2025.11.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/af/fe/b6045c782f1fd1ae317d2a6ca1884857ce5c20f59befe6ab25a8603c43a7/ruamel_yaml-0.18.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/06/c4/c124fbcef0684fcf3c9b72374c2a8c35c94464d8694c50f37eef27f5a145/ruamel_yaml_clib-0.2.15-cp313-cp313-win_amd64.whl @@ -1226,7 +1451,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/6e/d23bcde21d4ef0250a74e7505d2990d429f862be65810a3b650a69def7f0/ShExJSG-0.8.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl @@ -1246,16 +1470,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/9f/e7/84381106a701c812652405f4d74093f6da7c2748a639084fd2c092cd2c78/whenever-0.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d5/e4/62a677feefde05b12a70a4fc9bdc8558010182a801fbcab68cb56c2b0986/xarray-2025.12.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl - pypi: ./src/ca_biositing/datamodels - pypi: ./src/ca_biositing/pipeline - pypi: ./src/ca_biositing/webservice @@ -8782,6 +9002,24 @@ packages: - trio>=0.32.0 ; python_full_version >= '3.10' and extra == 'trio' - trio>=0.31.0 ; python_full_version < '3.10' and extra == 'trio' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.0-pyhcf101f3_0.conda + sha256: 830fc81970cd9d19869909b9b16d241f4d557e4f201a1030aa6ed87c6aa8b930 + md5: 9958d4a1ee7e9c768fe8f4fb51bd07ea + depends: + - exceptiongroup >=1.0.2 + - idna >=2.8 + - python >=3.10 + - typing_extensions >=4.5 + - python + constrains: + - trio >=0.32.0 + - uvloop >=0.21 + license: MIT + license_family: MIT + purls: + - pkg:pypi/anyio?source=hash-mapping + size: 144702 + timestamp: 1764375386926 - conda: https://conda.anaconda.org/conda-forge/osx-64/aom-3.9.1-hf036a51_0.conda sha256: 3032f2f55d6eceb10d53217c2a7f43e1eac83603d91e21ce502e8179e63a75f5 md5: 3f17bc32cb7fcb2b4bf3d8d37f656eb8 @@ -8873,6 +9111,96 @@ packages: - pywin32 ; extra == 'windows' - tzdata ; extra == 'windows' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda + sha256: bea62005badcb98b1ae1796ec5d70ea0fc9539e7d59708ac4e7d41e2f4bb0bad + md5: 8ac12aff0860280ee0cff7fa2cf63f3b + depends: + - argon2-cffi-bindings + - python >=3.9 + - typing-extensions + constrains: + - argon2_cffi ==999 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi?source=hash-mapping + size: 18715 + timestamp: 1749017288144 +- conda: https://conda.anaconda.org/conda-forge/linux-64/argon2-cffi-bindings-25.1.0-py312h4c3975b_2.conda + sha256: 7988c207b2b766dad5ebabf25a92b8d75cb8faed92f256fd7a4e0875c9ec6d58 + md5: 1567f06d717246abab170736af8bad1b + depends: + - __glibc >=2.17,<3.0.a0 + - cffi >=1.0.1 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 35646 + timestamp: 1762509443854 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/argon2-cffi-bindings-25.1.0-py312hcd1a082_2.conda + sha256: 4b9bc79be725463b1214ad2735657e9349865fa6980f0ccb892b63dd95d95449 + md5: ac6c4634dc5a01e92f604e55656d9158 + depends: + - cffi >=1.0.1 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python >=3.12,<3.13.0a0 *_cpython + - python_abi 3.12.* *_cp312 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 37418 + timestamp: 1762509514735 +- conda: https://conda.anaconda.org/conda-forge/osx-64/argon2-cffi-bindings-25.1.0-py313hf050af9_2.conda + sha256: e2644e87c26512e38c63ace8fc19120a472c0983718a8aa264862c25294d0632 + md5: 1fedb53ffc72b7d1162daa934ad7996b + depends: + - __osx >=10.13 + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 33301 + timestamp: 1762509795647 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/argon2-cffi-bindings-25.1.0-py313h6535dbc_2.conda + sha256: 05ea6fa7109235cfb4fc24526bae1fe82d88bbb5e697ab3945c313f5f041af5b + md5: e23e087109b2096db4cf9a3985bab329 + depends: + - __osx >=11.0 + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 33947 + timestamp: 1762510144907 +- conda: https://conda.anaconda.org/conda-forge/win-64/argon2-cffi-bindings-25.1.0-py313h5ea7bf4_2.conda + sha256: 3f8a1affdfeb2be5289d709e365fc6e386d734773895215cf8cbc5100fa6af9a + md5: eabb4b677b54874d7d6ab775fdaa3d27 + depends: + - cffi >=1.0.1 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: + - pkg:pypi/argon2-cffi-bindings?source=hash-mapping + size: 38779 + timestamp: 1762509796090 - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl name: arrow version: 1.4.0 @@ -8894,6 +9222,20 @@ packages: - pytz==2025.2 ; extra == 'test' - simplejson==3.* ; extra == 'test' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.4.0-pyhcf101f3_0.conda + sha256: 792da8131b1b53ff667bd6fc617ea9087b570305ccb9913deb36b8e12b3b5141 + md5: 85c4f19f377424eafc4ed7911b291642 + depends: + - python >=3.10 + - python-dateutil >=2.7.0 + - python-tzdata + - python + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/arrow?source=hash-mapping + size: 113854 + timestamp: 1760831179410 - pypi: https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl name: asgi-lifespan version: 2.1.0 @@ -8927,6 +9269,19 @@ packages: - pkg:pypi/asttokens?source=hash-mapping size: 28797 timestamp: 1763410017955 +- conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda + sha256: 3b7233041e462d9eeb93ea1dfe7b18aca9c358832517072054bb8761df0c324b + md5: d9d0f99095a9bb7e3641bca8c6ad2ac7 + depends: + - python >=3.9 + - typing_extensions >=4.0.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/async-lru?source=hash-mapping + size: 17335 + timestamp: 1742153708859 - conda: https://conda.anaconda.org/conda-forge/noarch/async-timeout-5.0.1-pyhd8ed1ab_1.conda sha256: 33d12250c870e06c9a313c6663cfbf1c50380b73dfbbb6006688c3134b29b45a md5: 5d842988b11a8c3ab57fb70840c83d24 @@ -10758,6 +11113,43 @@ packages: - xarray ; python_full_version < '3.15' and extra == 'test-tox' - coverage>=5.5 ; extra == 'test-tox-coverage' requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.14.3-pyha770c72_0.conda + sha256: bf1e71c3c0a5b024e44ff928225a0874fc3c3356ec1a0b6fe719108e6d1288f6 + md5: 5267bef8efea4127aacd1f4e1f149b6e + depends: + - python >=3.10 + - soupsieve >=1.2 + - typing-extensions + license: MIT + license_family: MIT + purls: + - pkg:pypi/beautifulsoup4?source=hash-mapping + size: 90399 + timestamp: 1764520638652 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.3.0-pyhcf101f3_0.conda + sha256: e03ba1a2b93fe0383c57920a9dc6b4e0c2c7972a3f214d531ed3c21dc8f8c717 + md5: b1a27250d70881943cca0dd6b4ba0956 + depends: + - python >=3.10 + - webencodings + - python + constrains: + - tinycss >=1.1.0,<1.5 + license: Apache-2.0 AND MIT + purls: + - pkg:pypi/bleach?source=hash-mapping + size: 141952 + timestamp: 1763589981635 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.3.0-h5f6438b_0.conda + sha256: f85f6b2c7938d8c20c80ce5b7e6349fafbb49294641b5648273c5f892b150768 + md5: 08a03378bc5293c6f97637323802f480 + depends: + - bleach ==6.3.0 pyhcf101f3_0 + - tinycss2 + license: Apache-2.0 AND MIT + purls: [] + size: 4386 + timestamp: 1763589981639 - conda: https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda sha256: e7af5d1183b06a206192ff440e08db1c4e8b2ca1f8376ee45fb2f3a85d4ee45d md5: 2c2fae981fd2afd00812c92ac47d023d @@ -11317,6 +11709,28 @@ packages: purls: [] size: 152827 timestamp: 1762967310929 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + noarch: python + sha256: 561e6660f26c35d137ee150187d89767c988413c978e1b712d53f27ddf70ea17 + md5: 9b347a7ec10940d3f7941ff6c460b551 + depends: + - cached_property >=1.5.2,<1.5.3.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4134 + timestamp: 1615209571450 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 + sha256: 6dbf7a5070cc43d90a1e4c2ec0c541c69d8e30a0e25f50ce9f6e4a432e42c5d7 + md5: 576d629e47797577ab0f1b351297ef4a + depends: + - python >=3.6 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/cached-property?source=hash-mapping + size: 11065 + timestamp: 1615209567874 - pypi: https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl name: cachetools version: 6.2.1 @@ -12742,6 +13156,17 @@ packages: - pkg:pypi/decorator?source=hash-mapping size: 14129 timestamp: 1740385067843 +- conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 + sha256: 9717a059677553562a8f38ff07f3b9f61727bd614f505658b0a5ecbcf8df89be + md5: 961b3a227b437d82ad7054484cfa71b2 + depends: + - python >=3.6 + license: PSF-2.0 + license_family: PSF + purls: + - pkg:pypi/defusedxml?source=hash-mapping + size: 24062 + timestamp: 1615232388757 - pypi: https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl name: deprecated version: 1.3.1 @@ -13489,6 +13914,18 @@ packages: requires_dist: - cached-property>=1.3.0 ; python_full_version < '3.8' requires_python: '>=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4,<4' +- conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + sha256: 2509992ec2fd38ab27c7cdb42cf6cadc566a1cc0d1021a2673475d9fa87c6276 + md5: d3549fd50d450b6d9e7dddff25dd2110 + depends: + - cached-property >=1.3.0 + - python >=3.9,<4 + license: MPL-2.0 + license_family: MOZILLA + purls: + - pkg:pypi/fqdn?source=hash-mapping + size: 16705 + timestamp: 1733327494780 - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda sha256: bf8e4dffe46f7d25dc06f31038cacb01672c47b9f45201f065b0f4d00ab0a83e md5: 4afc585cd97ba8a23809406cd8a9eda8 @@ -14972,6 +15409,18 @@ packages: version: 0.16.0 sha256: 63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/h11-0.16.0-pyhd8ed1ab_0.conda + sha256: f64b68148c478c3bfc8f8d519541de7d2616bf59d44485a5271041d40c061887 + md5: 4b69232755285701bc86a5afe4d9933a + depends: + - python >=3.9 + - typing_extensions + license: MIT + license_family: MIT + purls: + - pkg:pypi/h11?source=hash-mapping + size: 37697 + timestamp: 1745526482242 - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda sha256: 84c64443368f84b600bfecc529a1194a3b14c3656ee2e832d15a20e0329b6da3 md5: 164fc43f0b53b6e3a7bc7dce5e4f1dc9 @@ -15222,6 +15671,23 @@ packages: - socksio==1.* ; extra == 'socks' - trio>=0.22.0,<1.0 ; extra == 'trio' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda + sha256: 04d49cb3c42714ce533a8553986e1642d0549a05dc5cc48e0d43ff5be6679a5b + md5: 4f14640d58e2cc0aa0819d9d8ba125bb + depends: + - python >=3.9 + - h11 >=0.16 + - h2 >=3,<5 + - sniffio 1.* + - anyio >=4.0,<5.0 + - certifi + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/httpcore?source=hash-mapping + size: 49483 + timestamp: 1745602916758 - conda: https://conda.anaconda.org/conda-forge/noarch/httplib2-0.22.0-pyhd8ed1ab_1.conda sha256: f6ee2956004d98a6ea9ebdaa975dd222b658a31de3ed89348eb67749a0949c6f md5: 2a534e4b838b6efab1732c869d993b45 @@ -15264,6 +15730,21 @@ packages: - socksio==1.* ; extra == 'socks' - zstandard>=0.18.0 ; extra == 'zstd' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda + sha256: cd0f1de3697b252df95f98383e9edb1d00386bfdd03fdf607fa42fe5fcb09950 + md5: d6989ead454181f4f9bc987d3dc4e285 + depends: + - anyio + - certifi + - httpcore 1.* + - idna + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/httpx?source=hash-mapping + size: 63082 + timestamp: 1733663449209 - pypi: https://files.pythonhosted.org/packages/c3/5b/9512c5fb6c8218332b530f13500c6ff5f3ce3342f35e0dd7be9ac3856fd3/humanize-4.14.0-py3-none-any.whl name: humanize version: 4.14.0 @@ -15442,6 +15923,20 @@ packages: - pkg:pypi/importlib-metadata?source=hash-mapping size: 34641 timestamp: 1747934053147 +- conda: https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda + sha256: acc1d991837c0afb67c75b77fdc72b4bf022aac71fedd8b9ea45918ac9b08a80 + md5: c85c76dc67d75619a92f51dfbce06992 + depends: + - python >=3.9 + - zipp >=3.1.0 + constrains: + - importlib-resources >=6.5.2,<6.5.3.0a0 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/importlib-resources?source=hash-mapping + size: 33781 + timestamp: 1736252433366 - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.3.0-pyhd8ed1ab_0.conda sha256: e1a9e3b1c8fe62dc3932a616c284b5d8cbe3124bbfbedcf4ce5c828cb166ee19 md5: 9614359868482abba1bd15ce465e3c42 @@ -15608,6 +16103,22 @@ packages: - pkg:pypi/ipython-pygments-lexers?source=hash-mapping size: 13993 timestamp: 1737123723464 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.8-pyhd8ed1ab_0.conda + sha256: 6bb58afb7eabc8b4ac0c7e92707fb498313cc0164cf04e7ba1090dbf49af514b + md5: d68e3f70d1f068f1b66d94822fdc644e + depends: + - comm >=0.1.3 + - ipython >=6.1.0 + - jupyterlab_widgets >=3.0.15,<3.1.0 + - python >=3.10 + - traitlets >=4.3.1 + - widgetsnbextension >=4.0.14,<4.1.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipywidgets?source=hash-mapping + size: 114376 + timestamp: 1762040524661 - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl name: isodate version: 0.7.2 @@ -15620,6 +16131,18 @@ packages: requires_dist: - arrow>=0.15.0 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda + sha256: 08e838d29c134a7684bca0468401d26840f41c92267c4126d7b43a6b533b0aed + md5: 0b0154421989637d424ccf0f104be51a + depends: + - arrow >=0.15.0 + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/isoduration?source=hash-mapping + size: 19832 + timestamp: 1733493720346 - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda sha256: 92c4d217e2dc68983f724aa983cca5464dcb929c566627b26a2511159667dba8 md5: a4f4c5dc9b80bc50e0d3dc4e6e8f1bd9 @@ -15721,6 +16244,16 @@ packages: - click - pyyaml requires_python: '>=3.7.0' +- conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.13.0-pyhd8ed1ab_0.conda + sha256: ba03ca5a6db38d9f48bd30172e8c512dea7a686a5c7701c6fcdb7b3023dae2ad + md5: 8d5f66ebf832c4ce28d5c37a0e76605c + depends: + - python >=3.10 + license: Apache-2.0 + purls: + - pkg:pypi/json5?source=compressed-mapping + size: 34017 + timestamp: 1767325114901 - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl name: jsonasobj version: 1.3.1 @@ -15744,6 +16277,18 @@ packages: version: 3.0.0 sha256: 13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonpointer-3.0.0-pyhcf101f3_3.conda + sha256: 1a1328476d14dfa8b84dbacb7f7cd7051c175498406dc513ca6c679dc44f3981 + md5: cd2214824e36b0180141d422aba01938 + depends: + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jsonpointer?source=hash-mapping + size: 13967 + timestamp: 1765026384757 - pypi: https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl name: jsonschema version: 4.25.1 @@ -15771,6 +16316,22 @@ packages: - uri-template ; extra == 'format-nongpl' - webcolors>=24.6.0 ; extra == 'format-nongpl' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.1-pyhe01879c_0.conda + sha256: ac377ef7762e49cb9c4f985f1281eeff471e9adc3402526eea78e6ac6589cf1d + md5: 341fd940c242cf33e832c0402face56f + depends: + - attrs >=22.2.0 + - jsonschema-specifications >=2023.3.6 + - python >=3.9 + - referencing >=0.28.4 + - rpds-py >=0.7.1 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/jsonschema?source=hash-mapping + size: 81688 + timestamp: 1755595646123 - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl name: jsonschema-specifications version: 2025.9.1 @@ -15778,6 +16339,69 @@ packages: requires_dist: - referencing>=0.31.0 requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.9.1-pyhcf101f3_0.conda + sha256: 0a4f3b132f0faca10c89fdf3b60e15abb62ded6fa80aebfc007d05965192aa04 + md5: 439cd0f567d697b20a8f45cb70a1005a + depends: + - python >=3.10 + - referencing >=0.31.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/jsonschema-specifications?source=hash-mapping + size: 19236 + timestamp: 1757335715225 +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.1-he01879c_0.conda + sha256: aef6705fe1335e6472e1b6365fcdb586356b18dceff72d8d6a315fc90e900ccf + md5: 13e31c573c884962318a738405ca3487 + depends: + - jsonschema >=4.25.1,<4.25.2.0a0 + - fqdn + - idna + - isoduration + - jsonpointer >1.13 + - rfc3339-validator + - rfc3986-validator >0.1.0 + - rfc3987-syntax >=1.1.0 + - uri-template + - webcolors >=24.6.0 + license: MIT + license_family: MIT + purls: [] + size: 4744 + timestamp: 1755595646123 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + sha256: b538e15067d05768d1c0532a6d9b0625922a1cce751dd6a2af04f7233a1a70e9 + md5: 9453512288d20847de4356327d0e1282 + depends: + - ipykernel + - ipywidgets + - jupyter_console + - jupyterlab + - nbconvert-core + - notebook + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter?source=hash-mapping + size: 8891 + timestamp: 1733818677113 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.3.0-pyhcf101f3_0.conda + sha256: 897ad2e2c2335ef3c2826d7805e16002a1fd0d509b4ae0bc66617f0e0ff07bc2 + md5: 62b7c96c6cd77f8173cc5cada6a9acaa + depends: + - importlib-metadata >=4.8.3 + - jupyter_server >=1.1.2 + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-lsp?source=hash-mapping + size: 60377 + timestamp: 1756388269267 - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda sha256: 19d8bd5bb2fde910ec59e081eeb59529491995ce0d653a5209366611023a0b3a md5: 4ebae00eae9705b0c3d6d1018a81d047 @@ -15812,6 +16436,25 @@ packages: - pkg:pypi/jupyter-client?source=compressed-mapping size: 111367 timestamp: 1765375773813 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda + sha256: aee0cdd0cb2b9321d28450aec4e0fd43566efcd79e862d70ce49a68bf0539bcd + md5: 801dbf535ec26508fac6d4b24adfb76e + depends: + - ipykernel >=6.14 + - ipython + - jupyter_client >=7.0.0 + - jupyter_core >=4.12,!=5.0.* + - prompt_toolkit >=3.0.30 + - pygments + - python >=3.9 + - pyzmq >=17 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-console?source=hash-mapping + size: 26874 + timestamp: 1733818130068 - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyh6dadd2b_0.conda sha256: ed709a6c25b731e01563521ef338b93986cd14b5bc17f35e9382000864872ccc md5: a8db462b01221e9f5135be466faeb3e0 @@ -15848,6 +16491,140 @@ packages: - pkg:pypi/jupyter-core?source=hash-mapping size: 65503 timestamp: 1760643864586 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + sha256: 37e6ac3ccf7afcc730c3b93cb91a13b9ae827fd306f35dd28f958a74a14878b5 + md5: f56000b36f09ab7533877e695e4e8cb0 + depends: + - jsonschema-with-format-nongpl >=4.18.0 + - packaging + - python >=3.9 + - python-json-logger >=2.0.4 + - pyyaml >=5.3 + - referencing + - rfc3339-validator + - rfc3986-validator >=0.1.1 + - traitlets >=5.3 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-events?source=hash-mapping + size: 23647 + timestamp: 1738765986736 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.17.0-pyhcf101f3_0.conda + sha256: 74c4e642be97c538dae1895f7052599dfd740d8bd251f727bce6453ce8d6cd9a + md5: d79a87dcfa726bcea8e61275feed6f83 + depends: + - anyio >=3.1.0 + - argon2-cffi >=21.1 + - jinja2 >=3.0.3 + - jupyter_client >=7.4.4 + - jupyter_core >=4.12,!=5.0.* + - jupyter_events >=0.11.0 + - jupyter_server_terminals >=0.4.4 + - nbconvert-core >=6.4.4 + - nbformat >=5.3.0 + - overrides >=5.0 + - packaging >=22.0 + - prometheus_client >=0.9 + - python >=3.10 + - pyzmq >=24 + - send2trash >=1.8.2 + - terminado >=0.8.3 + - tornado >=6.2.0 + - traitlets >=5.6.0 + - websocket-client >=1.7 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server?source=hash-mapping + size: 347094 + timestamp: 1755870522134 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + sha256: 0890fc79422191bc29edf17d7b42cff44ba254aa225d31eb30819f8772b775b8 + md5: 2d983ff1b82a1ccb6f2e9d8784bdd6bd + depends: + - python >=3.9 + - terminado >=0.8.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server-terminals?source=hash-mapping + size: 19711 + timestamp: 1733428049134 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.5.1-pyhd8ed1ab_0.conda + sha256: ac31a517238173eb565ba9f517b1f9437fba48035f1276a9c1190c145657cafd + md5: f8e8f8db45e1a946ce9b20b0f60b3111 + depends: + - async-lru >=1.0.0 + - httpx >=0.25.0,<1 + - ipykernel >=6.5.0,!=6.30.0 + - jinja2 >=3.0.3 + - jupyter-lsp >=2.0.0 + - jupyter_core + - jupyter_server >=2.4.0,<3 + - jupyterlab_server >=2.28.0,<3 + - notebook-shim >=0.2 + - packaging + - python >=3.10 + - setuptools >=41.1.0 + - tomli >=1.2.2 + - tornado >=6.2.0 + - traitlets + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab?source=hash-mapping + size: 8141875 + timestamp: 1765819955819 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + sha256: dc24b900742fdaf1e077d9a3458fd865711de80bca95fe3c6d46610c532c6ef0 + md5: fd312693df06da3578383232528c468d + depends: + - pygments >=2.4.1,<3 + - python >=3.9 + constrains: + - jupyterlab >=4.0.8,<5.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-pygments?source=hash-mapping + size: 18711 + timestamp: 1733328194037 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.28.0-pyhcf101f3_0.conda + sha256: 381d2d6a259a3be5f38a69463e0f6c5dcf1844ae113058007b51c3bef13a7cee + md5: a63877cb23de826b1620d3adfccc4014 + depends: + - babel >=2.10 + - jinja2 >=3.0.3 + - json5 >=0.9.0 + - jsonschema >=4.18 + - jupyter_server >=1.21,<3 + - packaging >=21.3 + - python >=3.10 + - requests >=2.31 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-server?source=hash-mapping + size: 51621 + timestamp: 1761145478692 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.16-pyhcf101f3_1.conda + sha256: 5c03de243d7ae6247f39a402f4785d95e61c3be79ef18738e8f17155585d31a8 + md5: dbf8b81974504fa51d34e436ca7ef389 + depends: + - python >=3.10 + - python + constrains: + - jupyterlab >=3,<5 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-widgets?source=hash-mapping + size: 216779 + timestamp: 1762267481404 - conda: https://conda.anaconda.org/conda-forge/linux-64/kealib-1.6.2-h74a6b89_0.conda sha256: 7a5f7b5f0be22b9221a0c2dbdb00b5131aadd5a0cbf73318a4e718c36fb93445 md5: fa0c6d118bbd2fe95ffcc419dc075f65 @@ -16055,6 +16832,17 @@ packages: purls: [] size: 604863 timestamp: 1664997611416 +- conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.3.1-pyhd8ed1ab_0.conda + sha256: 49570840fb15f5df5d4b4464db8ee43a6d643031a2bc70ef52120a52e3809699 + md5: 9b965c999135d43a3d0f7bd7d024e26a + depends: + - python >=3.10 + license: MIT + license_family: MIT + purls: + - pkg:pypi/lark?source=compressed-mapping + size: 94312 + timestamp: 1761596921009 - conda: https://conda.anaconda.org/conda-forge/linux-64/laz-perf-3.4.0-h00ab1b0_0.conda sha256: 4c130699ffcfddff7073ac079558d3e970b3b6198e096d586183e05d1ee714ac md5: 3c32b1aac6372685e9eb832b8d789b80 @@ -25730,6 +26518,21 @@ packages: - pkg:pypi/markupsafe?source=hash-mapping size: 24541 timestamp: 1759055509267 +- conda: https://conda.anaconda.org/conda-forge/osx-64/markupsafe-3.0.3-py313h0f4d31d_0.conda + sha256: 9c698da56e3bdae80be2a7bc0d19565971b36060155374d16fce14271c8b695c + md5: 884a82dc80ecd251e38d647808c424b3 + depends: + - __osx >=10.13 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + constrains: + - jinja2 >=3.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/markupsafe?source=hash-mapping + size: 25105 + timestamp: 1759055575973 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py312h5748b74_0.conda sha256: b6aadcee6a0b814a0cb721e90575cbbe911b17ec46542460a9416ed2ec1a568e md5: 82221456841d3014a175199e4792465b @@ -25746,6 +26549,22 @@ packages: - pkg:pypi/markupsafe?source=hash-mapping size: 25121 timestamp: 1759055677633 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/markupsafe-3.0.3-py313h7d74516_0.conda + sha256: e06902a1bf370fdd4ada0a8c81c504868fdb7e9971b72c6bd395aa4e5a497bd2 + md5: 3df5979cc0b761dda0053ffdb0bca3ea + depends: + - __osx >=11.0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + constrains: + - jinja2 >=3.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/markupsafe?source=hash-mapping + size: 25778 + timestamp: 1759055530601 - conda: https://conda.anaconda.org/conda-forge/win-64/markupsafe-3.0.3-py313hd650c13_0.conda sha256: 988d14095c1392e055fd75e24544da2db01ade73b0c2f99ddc8e2b8678ead4cc md5: 47eaaa4405741beb171ea6edc6eaf874 @@ -25931,6 +26750,19 @@ packages: purls: [] size: 86618 timestamp: 1746450788037 +- conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.2.0-pyhcf101f3_0.conda + sha256: d3fb4beb5e0a52b6cc33852c558e077e1bfe44df1159eb98332d69a264b14bae + md5: b11e360fc4de2b0035fc8aaa74f17fd6 + depends: + - python >=3.10 + - typing_extensions + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/mistune?source=compressed-mapping + size: 74250 + timestamp: 1766504456031 - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.6.1-pyhd8ed1ab_1.conda sha256: 902d2e251f9a7ffa7d86a3e62be5b2395e28614bd4dbe5f50acf921fd64a8c35 md5: 14661160be39d78f2b210f2cc2766059 @@ -26357,6 +27189,66 @@ packages: - fastnumbers>=2.0.0 ; extra == 'fast' - pyicu>=1.0.0 ; extra == 'icu' requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.4-pyhd8ed1ab_0.conda + sha256: 1b66960ee06874ddceeebe375d5f17fb5f393d025a09e15b830ad0c4fffb585b + md5: 00f5b8dafa842e0c27c1cd7296aa4875 + depends: + - jupyter_client >=6.1.12 + - jupyter_core >=4.12,!=5.0.* + - nbformat >=5.1 + - python >=3.8 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbclient?source=compressed-mapping + size: 28473 + timestamp: 1766485646962 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyhcf101f3_1.conda + sha256: 8f575e5c042b17f4677179a6ba474bdbe76573936d3d3e2aeb42b511b9cb1f3f + md5: cfc86ccc3b1de35d36ccaae4c50391f5 + depends: + - beautifulsoup4 + - bleach-with-css !=5.0.0 + - defusedxml + - importlib-metadata >=3.6 + - jinja2 >=3.0 + - jupyter_core >=4.7 + - jupyterlab_pygments + - markupsafe >=2.0 + - mistune >=2.0.3,<4 + - nbclient >=0.5.0 + - nbformat >=5.7 + - packaging + - pandocfilters >=1.4.1 + - pygments >=2.4.1 + - python >=3.10 + - traitlets >=5.1 + - python + constrains: + - pandoc >=2.9.2,<4.0.0 + - nbconvert ==7.16.6 *_1 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbconvert?source=compressed-mapping + size: 199273 + timestamp: 1760797634443 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda + sha256: 7a5bd30a2e7ddd7b85031a5e2e14f290898098dc85bea5b3a5bf147c25122838 + md5: bbe1963f1e47f594070ffe87cdf612ea + depends: + - jsonschema >=2.6 + - jupyter_core >=4.12,!=5.0.* + - python >=3.9 + - python-fastjsonschema >=2.15 + - traitlets >=5.1 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbformat?source=hash-mapping + size: 100945 + timestamp: 1733402844974 - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 md5: 47e340acb35de30501a76c7c799c41d7 @@ -26569,6 +27461,36 @@ packages: - pkg:pypi/nose2?source=hash-mapping size: 92820 timestamp: 1580623268425 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.5.1-pyhcf101f3_0.conda + sha256: 672ec7db73c8bfbacf9227c0c2287effdeded77b4d06373f2e498a310ce76a8c + md5: c984a8b773a34e38f5cf399b6d582e5c + depends: + - importlib_resources >=5.0 + - jupyter_server >=2.4.0,<3 + - jupyterlab >=4.5.1,<4.6 + - jupyterlab_server >=2.28.0,<3 + - notebook-shim >=0.2,<0.3 + - python >=3.10 + - tornado >=6.2.0 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook?source=hash-mapping + size: 10040377 + timestamp: 1765875192987 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda + sha256: 7b920e46b9f7a2d2aa6434222e5c8d739021dbc5cc75f32d124a8191d86f9056 + md5: e7f89ea5f7ea9401642758ff50a2d9c1 + depends: + - jupyter_server >=1.8,<3 + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook-shim?source=hash-mapping + size: 16817 + timestamp: 1733408419340 - conda: https://conda.anaconda.org/conda-forge/linux-64/nspr-4.37-h29cc59b_0.conda sha256: 472306630dcd49a221863b91bd89f5b8b81daf1b99adf1968c9f12021176d873 md5: d73ccc379297a67ed921bd55b38a6c6a @@ -27417,6 +28339,18 @@ packages: version: 3.11.5 sha256: 2b91126e7b470ff2e75746f6f6ee32b9ab67b7a93c8ba1d15d3a0caaf16ec875 requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda + sha256: 1840bd90d25d4930d60f57b4f38d4e0ae3f5b8db2819638709c36098c6ba770c + md5: e51f1e4089cad105b6cac64bd8166587 + depends: + - python >=3.9 + - typing_utils + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/overrides?source=hash-mapping + size: 30139 + timestamp: 1734587755455 - conda: https://conda.anaconda.org/conda-forge/noarch/owslib-0.34.1-pyhd8ed1ab_0.conda sha256: bd1dc309ec0d161cd1f65279cfc548c2b9dda7dc8cf44277b303923cfdf40386 md5: 819c0171e7092a83806a7ef9158f9158 @@ -28654,6 +29588,17 @@ packages: - pandas>=0.23 - xarray requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 + sha256: 2bb9ba9857f4774b85900c2562f7e711d08dd48e2add9bee4e1612fbee27e16f + md5: 457c2c8c08e54905d6954e79cb5b5db9 + depends: + - python !=3.0,!=3.1,!=3.2,!=3.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pandocfilters?source=hash-mapping + size: 11627 + timestamp: 1631603397334 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-4.0.0-pyhd8ed1ab_0.conda sha256: ce76d5a1fc6c7ef636cbdbf14ce2d601a1bfa0dd8d286507c1fd02546fccb94b md5: 1a884d2b1ea21abfb73911dcdb8342e4 @@ -29610,6 +30555,17 @@ packages: requires_dist: - twisted ; extra == 'twisted' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.23.1-pyhd8ed1ab_0.conda + sha256: 13dc67de68db151ff909f2c1d2486fa7e2d51355b25cee08d26ede1b62d48d40 + md5: a1e91db2d17fd258c64921cb38e6745a + depends: + - python >=3.10 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/prometheus-client?source=hash-mapping + size: 54592 + timestamp: 1758278323953 - conda: https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.52-pyha770c72_0.conda sha256: 4817651a276016f3838957bfdf963386438c70761e9faec7749d411635979bae md5: edb16f14d920fb3faf17f5ce582942d6 @@ -29624,6 +30580,16 @@ packages: - pkg:pypi/prompt-toolkit?source=hash-mapping size: 273927 timestamp: 1756321848365 +- conda: https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.52-hd8ed1ab_0.conda + sha256: e79922a360d7e620df978417dd033e66226e809961c3e659a193f978a75a9b0b + md5: 6d034d3a6093adbba7b24cb69c8c621e + depends: + - prompt-toolkit >=3.0.52,<3.0.53.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 7212 + timestamp: 1756321849562 - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.1.3-py312h5253ce2_0.conda sha256: 1b679202ebccf47be64509a4fc2a438a66229403257630621651b2886b882597 md5: 82ce56c5a4a55165aed95e04923ab363 @@ -30506,6 +31472,68 @@ packages: - pkg:pypi/pynacl?source=hash-mapping size: 1184528 timestamp: 1764063741665 +- conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-core-12.1-py313h07bcf3a_0.conda + sha256: 1e0edd34b804e20ba064dcebcfce3066d841ec812f29ed65902da7192af617d1 + md5: 6a2c3a617a70f97ca53b7b88461b1c27 + depends: + - __osx >=10.13 + - libffi >=3.5.2,<3.6.0a0 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - setuptools + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-core?source=hash-mapping + size: 491157 + timestamp: 1763151415674 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-core-12.1-py313h40b429f_0.conda + sha256: 307ca29ebf2317bd2561639b1ee0290fd8c03c3450fa302b9f9437d8df6a5280 + md5: 31a0a72f3466682d0ea2ebcbd7d319b8 + depends: + - __osx >=11.0 + - libffi >=3.5.2,<3.6.0a0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + - setuptools + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-core?source=hash-mapping + size: 481508 + timestamp: 1763152124940 +- conda: https://conda.anaconda.org/conda-forge/osx-64/pyobjc-framework-cocoa-12.1-py313hf669bc3_0.conda + sha256: 4761b8448bb9ecfcd9636a506104e6474e0f4cb73d108f2088997702ae984a00 + md5: 628b5ad83d6140fe4bfa937e2f357ed7 + depends: + - __osx >=10.13 + - libffi >=3.5.2,<3.6.0a0 + - pyobjc-core 12.1.* + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-framework-cocoa?source=hash-mapping + size: 374120 + timestamp: 1763160397755 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyobjc-framework-cocoa-12.1-py313hcc5defa_0.conda + sha256: 194e188d8119befc952d04157079733e2041a7a502d50340ddde632658799fdc + md5: a6d28c8fc266a3d3c3dae183e25c4d31 + depends: + - __osx >=11.0 + - libffi >=3.5.2,<3.6.0a0 + - pyobjc-core 12.1.* + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyobjc-framework-cocoa?source=hash-mapping + size: 376136 + timestamp: 1763160678792 - pypi: https://files.pythonhosted.org/packages/c8/23/c8dd17cbb1bd6614f306a983e260e31c01f3e8e8cc8954ba68749db6ae82/pyparsing-3.3.0-py3-none-any.whl name: pyparsing version: 3.3.0 @@ -31352,6 +32380,18 @@ packages: - pkg:pypi/python-dotenv?source=hash-mapping size: 26922 timestamp: 1761503229008 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda + sha256: df9aa74e9e28e8d1309274648aac08ec447a92512c33f61a8de0afa9ce32ebe8 + md5: 23029aae904a2ba587daba708208012f + depends: + - python >=3.9 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/fastjsonschema?source=hash-mapping + size: 244628 + timestamp: 1755304154927 - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda sha256: 59f17182813f8b23709b7d4cfda82c33b72dd007cb729efa0033c609fbd92122 md5: c20172b4c59fbe288fa50cdc1b693d73 @@ -31408,6 +32448,17 @@ packages: - mkdocs-literate-nav ; extra == 'dev' - mike ; extra == 'dev' requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + sha256: 4790787fe1f4e8da616edca4acf6a4f8ed4e7c6967aa31b920208fc8f95efcca + md5: a61bf9ec79426938ff785eb69dbb1960 + depends: + - python >=3.6 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/python-json-logger?source=hash-mapping + size: 13383 + timestamp: 1677079727691 - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl name: python-slugify version: 8.0.4 @@ -31544,6 +32595,22 @@ packages: purls: [] size: 4856 timestamp: 1646866525560 +- conda: https://conda.anaconda.org/conda-forge/win-64/pywinpty-2.0.15-py313h5813708_1.conda + sha256: d34a7cd0a4a7dc79662cb6005e01d630245d9a942e359eb4d94b2fb464ed2552 + md5: 8f01ed27e2baa455e753301218e054fd + depends: + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + - winpty + license: MIT + license_family: MIT + purls: + - pkg:pypi/pywinpty?source=hash-mapping + size: 216075 + timestamp: 1759556799508 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py312h8a5da7c_0.conda sha256: 1b3dc4c25c83093fff08b86a3574bc6b94ba355c8eba1f35d805c5e256455fc7 md5: fba10c2007c8b06f77c5a23ce3a635ad @@ -33180,6 +34247,21 @@ packages: - rpds-py>=0.7.0 - typing-extensions>=4.4.0 ; python_full_version < '3.13' requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/referencing-0.37.0-pyhcf101f3_0.conda + sha256: 0577eedfb347ff94d0f2fa6c052c502989b028216996b45c7f21236f25864414 + md5: 870293df500ca7e18bedefa5838a22ab + depends: + - attrs >=22.2.0 + - python >=3.10 + - rpds-py >=0.7.0 + - typing_extensions >=4.4.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/referencing?source=hash-mapping + size: 51788 + timestamp: 1760379115194 - pypi: https://files.pythonhosted.org/packages/4c/93/181070cd1aa2fa541ff2d3afcf763ceecd4937b34c615fa92765020a6c90/regex-2025.10.23-cp313-cp313-macosx_10_13_x86_64.whl name: regex version: 2025.10.23 @@ -33263,10 +34345,46 @@ packages: requires_dist: - six requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + sha256: 2e4372f600490a6e0b3bac60717278448e323cab1c0fecd5f43f7c56535a99c5 + md5: 36de09a8d3e5d5e6f4ee63af49e59706 + depends: + - python >=3.9 + - six + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3339-validator?source=hash-mapping + size: 10209 + timestamp: 1733600040800 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + sha256: 2a5b495a1de0f60f24d8a74578ebc23b24aa53279b1ad583755f223097c41c37 + md5: 912a71cc01012ee38e6b90ddd561e36f + depends: + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3986-validator?source=hash-mapping + size: 7818 + timestamp: 1598024297745 - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl name: rfc3987 version: 1.3.8 sha256: 10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + sha256: 70001ac24ee62058557783d9c5a7bbcfd97bd4911ef5440e3f7a576f9e43bc92 + md5: 7234f99325263a5af6d4cd195035e8f2 + depends: + - python >=3.9 + - lark >=1.2.2 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3987-syntax?source=hash-mapping + size: 22913 + timestamp: 1752876729969 - pypi: https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl name: rich version: 14.2.0 @@ -33331,6 +34449,83 @@ packages: version: 0.30.0 sha256: 806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15 requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.30.0-py312h868fb18_0.conda + sha256: 62f46e85caaba30b459da7dfcf3e5488ca24fd11675c33ce4367163ab191a42c + md5: 3ffc5a3572db8751c2f15bacf6a0e937 + depends: + - python + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python_abi 3.12.* *_cp312 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 383750 + timestamp: 1764543174231 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rpds-py-0.30.0-py312h75d7d99_0.conda + sha256: 51e7d33b71b30ac5ceab09cc35521eccdaf4e3897ca4c6eda1059fb82a91b285 + md5: 85212b0e327723285cc36efddd25e03d + depends: + - python + - libgcc >=14 + - python_abi 3.12.* *_cp312 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 380599 + timestamp: 1764543504405 +- conda: https://conda.anaconda.org/conda-forge/osx-64/rpds-py-0.30.0-py313hcc225dc_0.conda + sha256: 8955e67a30f44fbfd390374ba27f445b9e56818b023ccb8fe8f0cd00bec03caa + md5: 7c8790b86262342a2c4f4c9709cf61ae + depends: + - python + - __osx >=10.13 + - python_abi 3.13.* *_cp313 + constrains: + - __osx >=10.13 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 370868 + timestamp: 1764543169321 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/rpds-py-0.30.0-py313h2c089d5_0.conda + sha256: db63344f91e8bfe77703c6764aa9eeafb44d165e286053214722814eabda0264 + md5: 190c2d0d4e98ec97df48cdb74caf44d8 + depends: + - python + - __osx >=11.0 + - python 3.13.* *_cp313 + - python_abi 3.13.* *_cp313 + constrains: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 358961 + timestamp: 1764543165314 +- conda: https://conda.anaconda.org/conda-forge/win-64/rpds-py-0.30.0-py313hfbe8231_0.conda + sha256: 27bd383787c0df7a0a926b11014fd692d60d557398dcf1d50c55aa2378507114 + md5: 58ae648b12cfa6df3923b5fd219931cb + depends: + - python + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/rpds-py?source=hash-mapping + size: 243419 + timestamp: 1764543047271 - pypi: https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl name: rsa version: 4.9.1 @@ -33876,6 +35071,44 @@ packages: version: 3.0.4 sha256: 9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh5552912_0.conda + sha256: 5893e203cb099c784bf5b08d29944b5402beebcc361d55e54b676e9b355c7844 + md5: dcff6f8ea9e86a0bda978b88f89f2310 + depends: + - __osx + - pyobjc-framework-cocoa + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=compressed-mapping + size: 22782 + timestamp: 1767192019917 +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyh6dadd2b_0.conda + sha256: f154f702baf550de9c1e3517f110bb71a056df5645027c8d15b37f3ea33722cc + md5: 40df72e963d80a403c1861ae9428b13c + depends: + - __win + - pywin32 + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=hash-mapping + size: 22947 + timestamp: 1767192046046 +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-2.0.0-pyha191276_0.conda + sha256: 27cd93b4f848a1c8193a7b1b8e6e6d03321462e96997ce95ea1a39305f7ac7cb + md5: f2cc28627a451a28ddd5ef5ab0bf579d + depends: + - __linux + - python >=3.10 + - python + license: BSD-3-Clause + purls: + - pkg:pypi/send2trash?source=hash-mapping + size: 24215 + timestamp: 1767192001650 - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda sha256: 972560fcf9657058e3e1f97186cc94389144b46dbdf58c807ce62e83f977e863 md5: 4de79c071274a53dcaf2a8c749d1499e @@ -34154,6 +35387,17 @@ packages: version: 1.3.1 sha256: 2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda + sha256: dce518f45e24cd03f401cb0616917773159a210c19d601c5f2d4e0e5879d30ad + md5: 03fe290994c5e4ec17293cfb6bdce520 + depends: + - python >=3.10 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/sniffio?source=compressed-mapping + size: 15698 + timestamp: 1762941572482 - pypi: https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl name: snowballstemmer version: 3.0.1 @@ -34176,6 +35420,17 @@ packages: name: sortedcontainers version: 2.4.0 sha256: a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 +- conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.8.1-pyhd8ed1ab_0.conda + sha256: 4ba9b8c45862e54d05ed9a04cc6aab5a17756ab9865f57cbf2836e47144153d2 + md5: 7de28c27fe620a4f7dbfaea137c6232b + depends: + - python >=3.10 + license: MIT + license_family: MIT + purls: + - pkg:pypi/soupsieve?source=compressed-mapping + size: 37951 + timestamp: 1766075884412 - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl name: sparqlslurper version: 0.5.1 @@ -34994,6 +36249,36 @@ packages: purls: [] size: 155714 timestamp: 1762510341121 +- conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh6dadd2b_1.conda + sha256: b375e8df0d5710717c31e7c8e93c025c37fa3504aea325c7a55509f64e5d4340 + md5: e43ca10d61e55d0a8ec5d8c62474ec9e + depends: + - __win + - pywinpty >=1.1.0 + - python >=3.10 + - tornado >=6.1.0 + - python + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/terminado?source=hash-mapping + size: 23665 + timestamp: 1766513806974 +- conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda + sha256: 6b6727a13d1ca6a23de5e6686500d0669081a117736a87c8abf444d60c1e40eb + md5: 17b43cee5cc84969529d5d0b0309b2cb + depends: + - __unix + - ptyprocess + - python >=3.10 + - tornado >=6.1.0 + - python + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/terminado?source=hash-mapping + size: 24749 + timestamp: 1766513766867 - conda: https://conda.anaconda.org/conda-forge/noarch/testcontainers-4.13.2-pyhd8ed1ab_0.conda sha256: df6e7c890567e8f2f4721de6573e685bd917eab17fcf662990bb0ebea6008a6e md5: 5c433e9a89d0f7de90be138e89568bd4 @@ -35190,6 +36475,19 @@ packages: purls: [] size: 2858595 timestamp: 1766191991759 +- conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.5.1-pyhcf101f3_0.conda + sha256: 7c803480dbfb8b536b9bf6287fa2aa0a4f970f8c09075694174eb4550a4524cd + md5: c0d0b883e97906f7524e2aac94be0e0d + depends: + - python >=3.10 + - webencodings >=0.4 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/tinycss2?source=compressed-mapping + size: 30571 + timestamp: 1764621508086 - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda sha256: a84ff687119e6d8752346d1d408d5cf360dee0badd487a472aa8ddedfdc219e1 md5: a0116df4f4ed05c303811a837d5b39d8 @@ -35406,6 +36704,17 @@ packages: - pkg:pypi/typing-extensions?source=hash-mapping size: 51692 timestamp: 1756220668932 +- conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda + sha256: 3088d5d873411a56bf988eee774559335749aed6f6c28e07bf933256afb9eb6c + md5: f6d7aa696c67756a650e91e15e88223c + depends: + - python >=3.9 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/typing-utils?source=hash-mapping + size: 15183 + timestamp: 1733331395943 - conda: https://conda.anaconda.org/conda-forge/linux-64/tzcode-2025b-hb9d3cd8_0.conda sha256: 324976aab17bee85979761f89457b6a43c11bd93dcebf950d65576b2ab44dd94 md5: 83aa65f939a5cf4a82bfa510cbc38b3f @@ -35808,6 +37117,17 @@ packages: - flake8-use-fstring ; extra == 'dev' - pep8-naming ; extra == 'dev' requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda + sha256: e0eb6c8daf892b3056f08416a96d68b0a358b7c46b99c8a50481b22631a4dfc0 + md5: e7cb0f5745e4c5035a460248334af7eb + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/uri-template?source=hash-mapping + size: 23990 + timestamp: 1733323714454 - conda: https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda sha256: 2aad2aeff7c69a2d7eecd7b662eef756b27d6a6b96f3e2c2a7071340ce14543e md5: d71d3a66528853c0a1ac2c02d79a0284 @@ -36148,6 +37468,28 @@ packages: version: 25.10.0 sha256: 032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-25.10.0-pyhd8ed1ab_0.conda + sha256: 21f6c8a20fe050d09bfda3fb0a9c3493936ce7d6e1b3b5f8b01319ee46d6c6f6 + md5: 6639b6b0d8b5a284f027a2003669aa65 + depends: + - python >=3.10 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webcolors?source=hash-mapping + size: 18987 + timestamp: 1761899393153 +- conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda + sha256: 19ff205e138bb056a46f9e3839935a2e60bd1cf01c8241a5e172a422fed4f9c6 + md5: 2841eb5bfc75ce15e9a0054b98dcd64d + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webencodings?source=hash-mapping + size: 15496 + timestamp: 1733236131358 - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.9.0-pyhd8ed1ab_0.conda sha256: 42a2b61e393e61cdf75ced1f5f324a64af25f347d16c60b14117393a98656397 md5: 2f1ed718fcd829c184a6d4f0f2e07409 @@ -36260,6 +37602,17 @@ packages: - tzdata>=2020.1 ; sys_platform == 'win32' - tzlocal>=4.0 ; sys_platform != 'darwin' and sys_platform != 'linux' requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda + sha256: 826af5e2c09e5e45361fa19168f46ff524e7a766022615678c3a670c45895d9a + md5: dc257b7e7cad9b79c1dfba194e92297b + depends: + - python >=3.10 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/widgetsnbextension?source=hash-mapping + size: 889195 + timestamp: 1762040404362 - conda: https://conda.anaconda.org/conda-forge/noarch/win_inet_pton-1.1.0-pyh7428d3b_8.conda sha256: 93807369ab91f230cf9e6e2a237eaa812492fe00face5b38068735858fba954f md5: 46e441ba871f524e2b067929da3051c2 @@ -36271,6 +37624,13 @@ packages: - pkg:pypi/win-inet-pton?source=hash-mapping size: 9555 timestamp: 1733130678956 +- conda: https://conda.anaconda.org/conda-forge/win-64/winpty-0.4.3-4.tar.bz2 + sha256: 9df10c5b607dd30e05ba08cbd940009305c75db242476f4e845ea06008b0a283 + md5: 1cee351bf20b830d991dbe0bc8cd7dfe + license: MIT + license_family: MIT + purls: [] + size: 1176306 - conda: https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.17.3-py312h4c3975b_1.conda sha256: 8320d5af37eb8933e5d129884ea013b2687e75b08aff5216193df3378eaea92f md5: 8af3faf88325836e46c6cb79828e058c diff --git a/pixi.toml b/pixi.toml index 9f1deb56..345ae21c 100644 --- a/pixi.toml +++ b/pixi.toml @@ -242,6 +242,7 @@ frontend-build = { cmd = "npm run build", cwd = "frontend", description = "Build # Kernel Dependencies # This is to enable Jupyter Notebooks with IPython kernel [feature.kernel.dependencies] +jupyter = ">=1.0.0,<2" ipykernel = ">=7.1.0,<8" ipython = ">=9.0.0,<10" diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb new file mode 100644 index 00000000..18fb0314 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -0,0 +1,1787 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ETL Notebook for CA Biositing Project\n", + "\n", + "This notebook provides a documented walkthrough of the ETL (Extract, Transform, Load) process for the CA Biositing project. It is designed for interactive development and exploration before migrating logic into the production pipeline.\n", + "\n", + "It covers:\n", + "\n", + "1. **Setup**: Importing necessary libraries and establishing a connection to the database.\n", + "2. **Extraction**: Pulling raw data from Google Sheets.\n", + "3. **Cleaning**: Standardizing data types, handling missing values, and cleaning column names.\n", + "4. **Normalization**: Replacing human-readable names (e.g., \"Corn\") with database foreign key IDs (e.g., `resource_id: 1`).\n", + "5. **Utilities**: Common functions for data manipulation and analysis.\n", + "6. **Deployment Plan**: A step-by-step guide for moving the code from this notebook into the production ETL modules." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-04 08:53:57,422 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", + "2026-01-04 08:53:57,423 - INFO - Successfully imported all project modules.\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "import logging\n", + "from IPython.display import display\n", + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "\n", + "# --- Basic Logging Configuration for Notebook ---\n", + "# When running in a notebook, we use Python's standard logging.\n", + "# In the production pipeline, this will be replaced by Prefect's `get_run_logger()`\n", + "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n", + "logger = logging.getLogger()\n", + "\n", + "# --- Robustly find the project root ---\n", + "# This ensures that the notebook can be run from any directory within the project.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# Add the project root to the Python path to allow for module imports\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " logger.info(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " logger.info(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import project modules ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + " from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import replace_name_with_id_df\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " logger.info('Successfully imported all project modules.')\n", + "except ImportError as e:\n", + " logger.error(f'Failed to import project modules: {e}', exc_info=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Cleaning Function" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " \"\"\"Cleans and standardizes a DataFrame extracted from Google Sheets.\n", + "\n", + " This function performs several key operations:\n", + " 1. Cleans column names to a standard format (snake_case).\n", + " 2. Drops rows where essential columns ('repl_no', 'value') are empty.\n", + " 3. Coerces data types for numeric and datetime columns, handling errors gracefully.\n", + " 4. Converts remaining columns to the best possible data types.\n", + "\n", + " Args:\n", + " df (pd.DataFrame): The raw DataFrame.\n", + "\n", + " Returns:\n", + " pd.DataFrame: The cleaned DataFrame.\n", + " \"\"\"\n", + " logger.info('Starting DataFrame cleaning process.')\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.error('Input is not a pandas DataFrame.')\n", + " return None\n", + " \n", + " try:\n", + " # 1. Clean names and drop rows with missing essential data\n", + " df_cleaned = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + " logger.info(f'Dropped {len(df) - len(df_cleaned)} rows with missing values.')\n", + "\n", + " # 2. Coerce numeric types\n", + " df_cleaned['repl_no'] = pd.to_numeric(df_cleaned['repl_no'], errors='coerce').astype('Int32')\n", + " df_cleaned['value'] = pd.to_numeric(df_cleaned['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Coerce datetime types\n", + " if 'created_at' in df_cleaned.columns:\n", + " df_cleaned['created_at'] = pd.to_datetime(df_cleaned['created_at'], errors='coerce')\n", + " if 'updated_at' in df_cleaned.columns:\n", + " df_cleaned['updated_at'] = pd.to_datetime(df_cleaned['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert other dtypes to best possible\n", + " df_cleaned = df_cleaned.convert_dtypes()\n", + " logger.info('Successfully cleaned DataFrame.')\n", + " \n", + " \n", + " # 5. Convert all string data to lowercase\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + " logger.info('Converted all string data to lowercase.')\n", + " return df_cleaned\n", + "\n", + " except Exception as e:\n", + " logger.error(f'An error occurred during DataFrame cleaning: {e}', exc_info=True)\n", + " return None\n", + "\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Normalization Function" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_dataframes(dataframes, normalize_columns):\n", + " \"\"\"Normalizes a list of DataFrames by replacing name columns with foreign key IDs.\n", + "\n", + " This function iterates through a list of dataframes and, for each one, iterates\n", + " through a dictionary of columns that need to be normalized. It uses the \n", + " `replace_name_with_id_df` utility to look up or create the corresponding ID\n", + " in the database.\n", + "\n", + " Args:\n", + " dataframes (list[pd.DataFrame]): A list of DataFrames to normalize.\n", + " normalize_columns (dict): A dictionary mapping column names to SQLModel classes and attributes.\n", + "\n", + " Returns:\n", + " list[pd.DataFrame]: The list of normalized DataFrames.\n", + " \"\"\"\n", + " logger.info(f'Starting normalization process for {len(dataframes)} dataframes.')\n", + " normalized_dfs = []\n", + " try:\n", + " with Session(engine) as db:\n", + " for i, df in enumerate(dataframes):\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.warning(f'Item {i+1} is not a DataFrame, skipping.')\n", + " continue\n", + " \n", + " logger.info(f'Processing DataFrame #{i+1} with {len(df)} rows.')\n", + " df_normalized = df.copy()\n", + "\n", + " for df_col, (model, model_name_attr) in normalize_columns.items():\n", + " if df_col not in df_normalized.columns:\n", + " logger.warning(f\"Column '{df_col}' not in DataFrame #{i+1}. Skipping normalization for this column.\")\n", + " continue\n", + " \n", + " try:\n", + " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name='id',\n", + " final_column_name=f'{df_col}_id'\n", + " )\n", + " new_col_name = f'{df_col}_id'\n", + " num_nulls = df_normalized[new_col_name].isnull().sum()\n", + " logger.info(f\"Successfully normalized '{df_col}'. New column '{new_col_name}' contains {num_nulls} null values.\")\n", + " except Exception as e:\n", + " logger.error(f\"Error normalizing column '{df_col}' in DataFrame #{i+1}: {e}\", exc_info=True)\n", + " continue # Continue to the next column\n", + " \n", + " normalized_dfs.append(df_normalized)\n", + " logger.info(f'Finished processing DataFrame #{i+1}.')\n", + " \n", + " logger.info('Committing database session.')\n", + " db.commit()\n", + " logger.info('Database commit successful.')\n", + " except Exception as e:\n", + " logger.error(f'A critical error occurred during the database session: {e}', exc_info=True)\n", + " db.rollback()\n", + " logger.info('Database session rolled back.')\n", + " \n", + " return normalized_dfs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ETL Execution Example" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-04 08:58:52,849 - INFO - Starting data extraction...\n", + "2026-01-04 08:58:52,865 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:52,877 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:54,699 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:54,703 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:54,724 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:54,736 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:55,992 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:55,996 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:56,016 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-04 08:58:56,029 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-04 08:58:57,291 - INFO - Successfully extracted raw data.\n", + "2026-01-04 08:58:57,294 - INFO - Finished in state Completed()\n", + "2026-01-04 08:58:57,296 - INFO - Data extraction complete.\n", + "2026-01-04 08:58:57,299 - INFO - Starting data cleaning...\n", + "2026-01-04 08:58:57,299 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,302 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,309 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,314 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,315 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,316 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,319 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,320 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,321 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-04 08:58:57,325 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-04 08:58:57,335 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "2026-01-04 08:58:57,340 - INFO - Converted all string data to lowercase.\n", + "2026-01-04 08:58:57,342 - INFO - Data cleaning complete.\n", + "2026-01-04 08:58:57,342 - INFO - Starting data normalization...\n", + "2026-01-04 08:58:57,343 - INFO - Starting normalization process for 3 dataframes.\n", + "2026-01-04 08:58:57,343 - INFO - Processing DataFrame #1 with 759 rows.\n", + "2026-01-04 08:58:57,344 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,355 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,356 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,367 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,367 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,369 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,369 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,372 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,372 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,374 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,374 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,375 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,375 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,379 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,379 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,379 - INFO - Finished processing DataFrame #1.\n", + "2026-01-04 08:58:57,379 - INFO - Processing DataFrame #2 with 64 rows.\n", + "2026-01-04 08:58:57,380 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,384 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,384 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,388 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,388 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,392 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,392 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,396 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,396 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,398 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,398 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,399 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,400 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,403 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,403 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,403 - INFO - Finished processing DataFrame #2.\n", + "2026-01-04 08:58:57,403 - INFO - Processing DataFrame #3 with 390 rows.\n", + "2026-01-04 08:58:57,404 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-04 08:58:57,406 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-04 08:58:57,406 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-04 08:58:57,408 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-04 08:58:57,408 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-04 08:58:57,409 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-04 08:58:57,410 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-04 08:58:57,411 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-04 08:58:57,411 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-04 08:58:57,413 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-04 08:58:57,413 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-04 08:58:57,415 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-04 08:58:57,415 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-04 08:58:57,416 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-04 08:58:57,416 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", + "2026-01-04 08:58:57,417 - INFO - Finished processing DataFrame #3.\n", + "2026-01-04 08:58:57,417 - INFO - Committing database session.\n", + "2026-01-04 08:58:57,418 - INFO - Database commit successful.\n", + "2026-01-04 08:58:57,418 - INFO - Data normalization complete.\n", + "2026-01-04 08:58:57,418 - INFO - Displaying results of normalization...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 1 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prox_uuid_031", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "source_codename", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "float64", + "type": "float" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "e2d8eb1c-3fc8-4b97-99b0-df76b598d9df", + "rows": [ + [ + "0", + "d7965110-407f-e356-d41d-b3b9a2b7b7", + "(73)b7b7", + "oakleaf", + "4c", + "prox01xk", + "1.0", + "prox01xk(73)1", + "61.849998474121094", + "2024-10-02 10:31:01", + null, + "pass", + "not ready", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "1", + "c8fea984-2e9a-8def-55fb-1a9d7d9ba8", + "(73)9ba8", + "oakleaf", + "4c", + "prox01xk", + "2.0", + "prox01xk(73)2", + "63.209999084472656", + "2024-10-02 10:31:31", + null, + "pass", + "ready", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "2", + "df304d5d-3a85-4881-7142-6d4e5f957d", + "(73)957d", + "oakleaf", + "4c", + "prox01xk", + "3.0", + "prox01xk(73)3", + "63.27000045776367", + "2024-10-02 10:32:01", + null, + "pass", + "imported", + "", + "24", + "109", + "7", + "25", + "1", + "1", + "6" + ], + [ + "3", + "01c6c5be-cea6-54af-3924-b0bad69335", + "(73)9335", + "oakleaf", + "4c", + "prox01xk", + "1.0", + "prox01xk(73)1", + "0.6899999976158142", + "2024-10-03 10:31:01", + null, + "pass", + "import failed", + "", + "24", + "109", + "7", + "23", + "1", + "1", + "6" + ], + [ + "4", + "126745c7-dd41-2f6d-0dc5-28dbca415f", + "(73)415f", + "oakleaf", + "4c", + "prox01xk", + "2.0", + "prox01xk(73)2", + "0.8899999856948853", + "2024-10-03 10:31:31", + null, + "pass", + "", + "", + "24", + "109", + "7", + "23", + "1", + "1", + "6" + ] + ], + "shape": { + "columns": 20, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
prox_uuid_031record_idsource_codenamestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_resultupload_statusnoteresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
0d7965110-407f-e356-d41d-b3b9a2b7b7(73)b7b7oakleaf4cprox01xk1.0prox01xk(73)161.8499982024-10-02 10:31:01NaTpassnot ready24109725116
1c8fea984-2e9a-8def-55fb-1a9d7d9ba8(73)9ba8oakleaf4cprox01xk2.0prox01xk(73)263.2099992024-10-02 10:31:31NaTpassready24109725116
2df304d5d-3a85-4881-7142-6d4e5f957d(73)957doakleaf4cprox01xk3.0prox01xk(73)363.2700002024-10-02 10:32:01NaTpassimported24109725116
301c6c5be-cea6-54af-3924-b0bad69335(73)9335oakleaf4cprox01xk1.0prox01xk(73)10.6900002024-10-03 10:31:01NaTpassimport failed24109723116
4126745c7-dd41-2f6d-0dc5-28dbca415f(73)415foakleaf4cprox01xk2.0prox01xk(73)20.8900002024-10-03 10:31:31NaTpass24109723116
\n", + "
" + ], + "text/plain": [ + " prox_uuid_031 record_id source_codename storage_cond \\\n", + "0 d7965110-407f-e356-d41d-b3b9a2b7b7 (73)b7b7 oakleaf 4c \n", + "1 c8fea984-2e9a-8def-55fb-1a9d7d9ba8 (73)9ba8 oakleaf 4c \n", + "2 df304d5d-3a85-4881-7142-6d4e5f957d (73)957d oakleaf 4c \n", + "3 01c6c5be-cea6-54af-3924-b0bad69335 (73)9335 oakleaf 4c \n", + "4 126745c7-dd41-2f6d-0dc5-28dbca415f (73)415f oakleaf 4c \n", + "\n", + " exper_abbrev repl_no repl_id value created_at \\\n", + "0 prox01xk 1.0 prox01xk(73)1 61.849998 2024-10-02 10:31:01 \n", + "1 prox01xk 2.0 prox01xk(73)2 63.209999 2024-10-02 10:31:31 \n", + "2 prox01xk 3.0 prox01xk(73)3 63.270000 2024-10-02 10:32:01 \n", + "3 prox01xk 1.0 prox01xk(73)1 0.690000 2024-10-03 10:31:01 \n", + "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", + "\n", + " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", + "0 NaT pass not ready 24 109 \n", + "1 NaT pass ready 24 109 \n", + "2 NaT pass imported 24 109 \n", + "3 NaT pass import failed 24 109 \n", + "4 NaT pass 24 109 \n", + "\n", + " preparation_method_id parameter_id unit_id analyst_email_id \\\n", + "0 7 25 1 1 \n", + "1 7 25 1 1 \n", + "2 7 25 1 1 \n", + "3 7 23 1 1 \n", + "4 7 23 1 1 \n", + "\n", + " analysis_type_id \n", + "0 6 \n", + "1 6 \n", + "2 6 \n", + "3 6 \n", + "4 6 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 2 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ult_uuid_037", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "ult_sample_name", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "323960ff-b723-432d-b7c9-fa0f9dd8c185", + "rows": [ + [ + "0", + "421617a5-e50e-9642-2974-b3275fe822", + ")u22e822", + "hum-almhu023km2(15)u22", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(15)1", + "88.5999984741211", + null, + null, + "pass", + "1", + "", + "", + "", + "8", + "80", + "6", + "28", + "3", + "2", + "7" + ], + [ + "1", + "7e7919c2-5db4-6bef-75e2-7e51321200", + ")u001200", + "hum-almhu023km2(15)u00", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(15)1", + null, + null, + null, + "fail", + " 1 dup", + "", + "", + "", + "8", + "80", + "6", + "28", + "3", + "2", + "7" + ], + [ + "2", + "3aa85881-1185-642f-c44b-41ad2275d2", + ")ud275d2", + "hum-almsh022km2(13)ud2", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(13)1", + "81.30000305175781", + null, + null, + "pass", + "2", + "", + "", + "", + "26", + "96", + "6", + "28", + "3", + "2", + "7" + ], + [ + "3", + "fa418804-6c4f-4c90-d78f-84d7df54d3", + ")ud354d3", + "ene-wash017okm2(82)ud3", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(82)1", + "92.19999694824219", + null, + null, + "pass", + "3", + "", + "", + "", + "2", + "91", + "9", + "28", + "3", + "2", + "7" + ], + [ + "4", + "6fdacbfc-7e0b-473b-444f-85b7650267", + ")u670267", + "ebo-gppm010okm2(1b)u67", + "rt vacuum sealed", + "ult26kh", + "1", + "ult26kh(1b)1", + "93.80000305175781", + null, + null, + "pass", + "4", + "", + "", + "", + "7", + "153", + "9", + "28", + "3", + "2", + "7" + ] + ], + "shape": { + "columns": 22, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ult_uuid_037record_idult_sample_namestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_at...equipmentraw_data_urlupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
0421617a5-e50e-9642-2974-b3275fe822)u22e822hum-almhu023km2(15)u22rt vacuum sealedult26kh1ult26kh(15)188.599998NaTNaT...880628327
17e7919c2-5db4-6bef-75e2-7e51321200)u001200hum-almhu023km2(15)u00rt vacuum sealedult26kh1ult26kh(15)1NaNNaTNaT...880628327
23aa85881-1185-642f-c44b-41ad2275d2)ud275d2hum-almsh022km2(13)ud2rt vacuum sealedult26kh1ult26kh(13)181.300003NaTNaT...2696628327
3fa418804-6c4f-4c90-d78f-84d7df54d3)ud354d3ene-wash017okm2(82)ud3rt vacuum sealedult26kh1ult26kh(82)192.199997NaTNaT...291928327
46fdacbfc-7e0b-473b-444f-85b7650267)u670267ebo-gppm010okm2(1b)u67rt vacuum sealedult26kh1ult26kh(1b)193.800003NaTNaT...7153928327
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " ult_uuid_037 record_id ult_sample_name \\\n", + "0 421617a5-e50e-9642-2974-b3275fe822 )u22e822 hum-almhu023km2(15)u22 \n", + "1 7e7919c2-5db4-6bef-75e2-7e51321200 )u001200 hum-almhu023km2(15)u00 \n", + "2 3aa85881-1185-642f-c44b-41ad2275d2 )ud275d2 hum-almsh022km2(13)ud2 \n", + "3 fa418804-6c4f-4c90-d78f-84d7df54d3 )ud354d3 ene-wash017okm2(82)ud3 \n", + "4 6fdacbfc-7e0b-473b-444f-85b7650267 )u670267 ebo-gppm010okm2(1b)u67 \n", + "\n", + " storage_cond exper_abbrev repl_no repl_id value created_at \\\n", + "0 rt vacuum sealed ult26kh 1 ult26kh(15)1 88.599998 NaT \n", + "1 rt vacuum sealed ult26kh 1 ult26kh(15)1 NaN NaT \n", + "2 rt vacuum sealed ult26kh 1 ult26kh(13)1 81.300003 NaT \n", + "3 rt vacuum sealed ult26kh 1 ult26kh(82)1 92.199997 NaT \n", + "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", + "\n", + " updated_at ... equipment raw_data_url upload_status resource_id \\\n", + "0 NaT ... 8 \n", + "1 NaT ... 8 \n", + "2 NaT ... 26 \n", + "3 NaT ... 2 \n", + "4 NaT ... 7 \n", + "\n", + " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", + "0 80 6 28 3 \n", + "1 80 6 28 3 \n", + "2 96 6 28 3 \n", + "3 91 9 28 3 \n", + "4 153 9 28 3 \n", + "\n", + " analyst_email_id analysis_type_id \n", + "0 2 7 \n", + "1 2 7 \n", + "2 2 7 \n", + "3 2 7 \n", + "4 2 7 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Normalized DataFrame 3 ---\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", + "rawType": "object", + "type": "string" + }, + { + "name": "exper_abbrev", + "rawType": "object", + "type": "string" + }, + { + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", + "rawType": "object", + "type": "string" + }, + { + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "string" + }, + { + "name": "equipment", + "rawType": "object", + "type": "string" + }, + { + "name": "raw_data_url", + "rawType": "object", + "type": "string" + }, + { + "name": "upload_status", + "rawType": "object", + "type": "string" + }, + { + "name": "resource_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "prepared_sample_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "preparation_method_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "parameter_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "unit_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analyst_email_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "analysis_type_id", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "31e4fb08-8acb-41fc-bdec-4f79aea24d5f", + "rows": [ + [ + "0", + "3ee2993d-86e3-1f16-c7ea-f8d555e114", + "(85)e114", + "rt vacuum sealed", + "cmp04xk", + "1", + "cmp04xk(85)1", + "14.15999984741211", + "2025-01-23 09:00:01", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "1", + "46878ef9-1226-22a0-d5d8-cf65e241cb", + "(85)41cb", + "rt vacuum sealed", + "cmp04xk", + "2", + "cmp04xk(85)2", + "14.180000305175781", + "2025-01-23 09:00:16", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "2", + "76a7a2f4-c4e4-e60f-1187-dec6e02246", + "(85)2246", + "rt vacuum sealed", + "cmp04xk", + "3", + "cmp04xk(85)3", + "14.119999885559082", + "2025-01-23 09:00:31", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "12", + "2", + "1", + "3" + ], + [ + "3", + "7a136832-286b-07cb-62de-acf52f9311", + "(85)9311", + "rt vacuum sealed", + "cmp04xk", + "1", + "cmp04xk(85)1", + "15.739999771118164", + "2025-01-23 09:00:46", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "14", + "2", + "1", + "3" + ], + [ + "4", + "b709ecee-f9a6-a55d-a59e-93b7b863d7", + "(85)63d7", + "rt vacuum sealed", + "cmp04xk", + "2", + "cmp04xk(85)2", + "15.75", + "2025-01-23 09:01:01", + null, + "pass", + "", + "", + "", + "ready", + "24", + "92", + "8", + "14", + "2", + "1", + "3" + ] + ], + "shape": { + "columns": 21, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cmp_uuid_033record_idstorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_result...equipmentraw_data_urlupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
03ee2993d-86e3-1f16-c7ea-f8d555e114(85)e114rt vacuum sealedcmp04xk1cmp04xk(85)114.162025-01-23 09:00:01NaTpass...ready2492812213
146878ef9-1226-22a0-d5d8-cf65e241cb(85)41cbrt vacuum sealedcmp04xk2cmp04xk(85)214.182025-01-23 09:00:16NaTpass...ready2492812213
276a7a2f4-c4e4-e60f-1187-dec6e02246(85)2246rt vacuum sealedcmp04xk3cmp04xk(85)314.122025-01-23 09:00:31NaTpass...ready2492812213
37a136832-286b-07cb-62de-acf52f9311(85)9311rt vacuum sealedcmp04xk1cmp04xk(85)115.742025-01-23 09:00:46NaTpass...ready2492814213
4b709ecee-f9a6-a55d-a59e-93b7b863d7(85)63d7rt vacuum sealedcmp04xk2cmp04xk(85)215.752025-01-23 09:01:01NaTpass...ready2492814213
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " cmp_uuid_033 record_id storage_cond \\\n", + "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 rt vacuum sealed \n", + "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb rt vacuum sealed \n", + "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 rt vacuum sealed \n", + "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 rt vacuum sealed \n", + "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 rt vacuum sealed \n", + "\n", + " exper_abbrev repl_no repl_id value created_at updated_at \\\n", + "0 cmp04xk 1 cmp04xk(85)1 14.16 2025-01-23 09:00:01 NaT \n", + "1 cmp04xk 2 cmp04xk(85)2 14.18 2025-01-23 09:00:16 NaT \n", + "2 cmp04xk 3 cmp04xk(85)3 14.12 2025-01-23 09:00:31 NaT \n", + "3 cmp04xk 1 cmp04xk(85)1 15.74 2025-01-23 09:00:46 NaT \n", + "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", + "\n", + " qc_result ... equipment raw_data_url upload_status resource_id \\\n", + "0 pass ... ready 24 \n", + "1 pass ... ready 24 \n", + "2 pass ... ready 24 \n", + "3 pass ... ready 24 \n", + "4 pass ... ready 24 \n", + "\n", + " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", + "0 92 8 12 2 \n", + "1 92 8 12 2 \n", + "2 92 8 12 2 \n", + "3 92 8 14 2 \n", + "4 92 8 14 2 \n", + "\n", + " analyst_email_id analysis_type_id \n", + "0 1 3 \n", + "1 1 3 \n", + "2 1 3 \n", + "3 1 3 \n", + "4 1 3 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- 1. Extraction ---\n", + "# In a real Prefect flow, each extraction would be a separate task.\n", + "logger.info('Starting data extraction...')\n", + "prox_df = proximate.extract(project_root=project_root)\n", + "ult_df = ultimate.extract(project_root=project_root)\n", + "cmp_df = cmpana.extract(project_root=project_root)\n", + "dataframes = [prox_df, ult_df, cmp_df]\n", + "logger.info('Data extraction complete.')\n", + "\n", + "# --- 2. Cleaning ---\n", + "# This list comprehension applies the cleaning function to each extracted dataframe.\n", + "logger.info('Starting data cleaning...')\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes if df is not None]\n", + "logger.info('Data cleaning complete.')\n", + "\n", + "# --- 3. Normalization ---\n", + "# This dictionary defines the columns to be normalized. \n", + "# The key is the column name in the DataFrame.\n", + "# The value is a tuple containing the corresponding SQLAlchemy model and the name of the attribute on the model to match against.\n", + "NORMALIZE_COLUMNS = {\n", + " 'resource': (Resource, 'name'),\n", + " 'prepared_sample': (PreparedSample, 'name'),\n", + " 'preparation_method': (PreparationMethod, 'name'),\n", + " 'parameter': (Parameter, 'name'),\n", + " 'unit': (Unit, 'name'),\n", + " 'analyst_email': (Contact, 'email'),\n", + " 'analysis_type': (AnalysisType, 'name'),\n", + " 'primary_ag_product': (PrimaryAgProduct, 'name')\n", + "}\n", + "\n", + "logger.info('Starting data normalization...')\n", + "normalized_dataframes = normalize_dataframes(clean_dataframes, NORMALIZE_COLUMNS)\n", + "logger.info('Data normalization complete.')\n", + "\n", + "# --- 4. Display Results ---\n", + "logger.info('Displaying results of normalization...')\n", + "for i, df in enumerate(normalized_dataframes):\n", + " print(f'--- Normalized DataFrame {i+1} ---')\n", + " display(df.head())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deployment Plan\n", + "\n", + "The code in this notebook will be transitioned to the main ETL pipeline by following these steps:\n", + "\n", + "1. **Function Migration**: The `clean_the_gsheets` and `normalize_dataframes` functions will be moved to a new utility module, for example, `src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_utils.py`. Each function will be decorated with `@task` from Prefect to turn it into a reusable pipeline component.\n", + "2. **Flow Creation**: A new Prefect flow will be created in the `src/ca_biositing/pipeline/ca_biositing/pipeline/flows/` directory (e.g., `master_extraction_flow.py`). This flow will orchestrate the entire ETL process for a given data source.\n", + "3. **Task Integration**: The new flow will be composed of individual tasks. It will call the existing extraction tasks (`proximate.extract`, etc.), and then pass the results to the new cleaning and normalization tasks from `etl_utils.py`.\n", + "4. **Logging**: The `logging` module will be replaced with `get_run_logger()` from Prefect within the tasks to ensure logs are captured by the Prefect UI.\n", + "5. **Configuration**: The `NORMALIZE_COLUMNS` dictionary will be moved to a configuration file or defined within the relevant flow to make it easier to manage and modify without changing the code.\n", + "6. **Testing**: Unit tests will be written for the new utility functions in `etl_utils.py`. An integration test will be created for the new Prefect flow to ensure all the tasks work together correctly.\n", + "7. **Deployment**: Once the flow is complete and tested, it will be deployed to the Prefect server using the `pixi run deploy` command, making it available to be run on a schedule or manually via the UI." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Pixi Default Environment", + "language": "python", + "name": "pixi-default" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb index 9d27799e..bfb3c8c4 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -2,62 +2,130 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "from IPython.display import display\n", + "\n", + "# --- Robustly find the project root ---\n", + "# The project root is the directory containing the 'pixi.toml' file.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path): # Stop at the filesystem root\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# --- Add project root to sys.path ---\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " print(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " print(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import the module ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " print(\"Successfully imported all module.\")\n", + "except ImportError as e:\n", + " print(f\"Failed to import modules: {e}\")\n", + " print(f\"\\nFull sys.path: {sys.path}\")\n", + "\n", + "# --- Run the extraction ---\n", + "if 'proximate' in locals():\n", + " try:\n", + " # Pass the project_root to the extract function\n", + " df = proximate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted proximate data.\")\n", + " display(df.head())\n", + " else:\n", + " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", + "\n", + "if 'ultimate' in locals():\n", + " try:\n", + " df2 = ultimate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted Ultimate data.\")\n", + " display(df2.head())\n", + " else:\n", + " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during extraction: {e}\")\n", + "\n", + "if 'cmpana' in locals():\n", + " try:\n", + " df3 = cmpana.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted CmpAna data.\")\n", + " display(df3.head())\n", + " else:\n", + " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", + " finally:\n", + " print(\"\\nCmp Ana extraction process completed.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " # 1. Clean names and drop rows\n", + " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + "\n", + " # 2. Coerce types (using errors='coerce' handles messy string data)\n", + " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", + " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Dates\n", + " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", + " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert remaining objects to best possible types (like strings)\n", + " df = df.convert_dtypes()\n", + "\n", + " # 5. Convert all string data to lowercase\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "\n", + " # 6. Convert empty strings to NaN\n", + " df.replace(\"\", np.nan, inplace=True)\n", + "\n", + " return df # Return the FULL dataframe, not just .head()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", - "Successfully imported all module.\n" - ] - }, - { - "data": { - "text/html": [ - "
20:54:42.753 | INFO    | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:42.753 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.230 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.230 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.234 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.234 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted proximate data.\n" + "Connected to database.\n" ] }, { @@ -70,228 +138,182 @@ "type": "integer" }, { - "name": "Prox_UUID_031", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Source_codename", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Parameter", - "rawType": "object", - "type": "string" - }, - { - "name": "Value", - "rawType": "object", - "type": "string" - }, - { - "name": "Unit", - "rawType": "object", - "type": "string" - }, - { - "name": "Created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "Updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "QC_result", - "rawType": "object", - "type": "string" + "name": "id", + "rawType": "int64", + "type": "integer" }, { - "name": "Upload_status", + "name": "name", "rawType": "object", "type": "string" }, { - "name": "Note", + "name": "note", "rawType": "object", - "type": "string" + "type": "unknown" }, { - "name": "Analysis_type", + "name": "description", "rawType": "object", - "type": "string" + "type": "unknown" }, { - "name": "Analyst_email", + "name": "uri", "rawType": "object", - "type": "string" + "type": "unknown" } ], - "ref": "b6c1f313-06e4-4dba-8a9e-6a1dc878820f", + "ref": "f594ce12-31fc-400e-807f-9ae024f154ba", "rows": [ [ "0", - "D7965110-407F-E356-D41D-B3B9A2B7B7", - "(73)B7B7", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "1", - "Prox01xk(73)1", - "Moisture", - "61.85", - "% total weight", - "2024-10-02 10:31:01", - "", - "Pass", - "not ready", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Tomatoes for processing", + null, + null, + null ], [ "1", - "C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8", - "(73)9BA8", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "2", - "Prox01xk(73)2", - "Moisture", - "63.21", - "% total weight", - "2024-10-02 10:31:31", - "", - "Pass", - "ready", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Grapes", + null, + null, + null ], [ "2", - "DF304D5D-3A85-4881-7142-6D4E5F957D", - "(73)957D", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", "3", - "Prox01xk(73)3", - "Moisture", - "63.27", - "% total weight", - "2024-10-02 10:32:01", - "", - "Pass", - "imported", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "Almonds", + null, + null, + null ], [ "3", - "01C6C5BE-CEA6-54AF-3924-B0BAD69335", - "(73)9335", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", - "1", - "Prox01xk(73)1", - "Ash", - "0.69", - "% total weight", - "2024-10-03 10:31:01", - "", - "Pass", - "import failed", - "", - "Proximate analysis", - "xkang2@lbl.gov" + "4", + "Walnuts", + null, + null, + null ], [ "4", - "126745C7-DD41-2F6D-0DC5-28DBCA415F", - "(73)415F", - "Oakleaf", - "Oak-TmPm01A(73)", - "Tomato pomace", - "As Is", - "4C", - "Prox01xk", - "2", - "Prox01xk(73)2", - "Ash", - "0.89", - "% total weight", - "2024-10-03 10:31:31", - "", - "Pass", - "", - "", - "Proximate analysis", - "xkang2@lbl.gov" - ] - ], - "shape": { - "columns": 20, - "rows": 5 - } + "5", + "Sweet potatoes", + null, + null, + null + ], + [ + "5", + "6", + "Algae", + null, + null, + null + ], + [ + "6", + "7", + "Olives - processing", + null, + null, + null + ], + [ + "7", + "8", + "Corn - all", + null, + null, + null + ], + [ + "8", + "9", + "Hay - alfalfa", + null, + null, + null + ], + [ + "9", + "10", + "Silage - wheat", + null, + null, + null + ], + [ + "10", + "11", + "Rice", + null, + null, + null + ], + [ + "11", + "12", + "Peaches", + null, + null, + null + ], + [ + "12", + "13", + "Potatoes", + null, + null, + null + ], + [ + "13", + "14", + "Cucumbers and pickles", + null, + null, + null + ], + [ + "14", + "15", + "Pistachios", + null, + null, + null + ], + [ + "15", + "16", + "Cotton", + null, + null, + null + ], + [ + "16", + "17", + "Olives - market", + null, + null, + null + ], + [ + "17", + "18", + "", + null, + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 18 + } }, "text/html": [ "
\n", @@ -312,3436 +334,527 @@ " \n", " \n", " \n", - " Prox_UUID_031\n", - " Record_ID\n", - " Source_codename\n", - " Prepared_sample\n", - " Resource\n", - " Preparation_method\n", - " Storage_cond\n", - " Exper_abbrev\n", - " Repl_no\n", - " Repl_ID\n", - " Parameter\n", - " Value\n", - " Unit\n", - " Created_at\n", - " Updated_at\n", - " QC_result\n", - " Upload_status\n", - " Note\n", - " Analysis_type\n", - " Analyst_email\n", + " id\n", + " name\n", + " note\n", + " description\n", + " uri\n", " \n", " \n", " \n", " \n", " 0\n", - " D7965110-407F-E356-D41D-B3B9A2B7B7\n", - " (73)B7B7\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 1\n", - " Prox01xk(73)1\n", - " Moisture\n", - " 61.85\n", - " % total weight\n", - " 2024-10-02 10:31:01\n", - " \n", - " Pass\n", - " not ready\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Tomatoes for processing\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 1\n", - " C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8\n", - " (73)9BA8\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 2\n", - " Prox01xk(73)2\n", - " Moisture\n", - " 63.21\n", - " % total weight\n", - " 2024-10-02 10:31:31\n", - " \n", - " Pass\n", - " ready\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Grapes\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 2\n", - " DF304D5D-3A85-4881-7142-6D4E5F957D\n", - " (73)957D\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", " 3\n", - " Prox01xk(73)3\n", - " Moisture\n", - " 63.27\n", - " % total weight\n", - " 2024-10-02 10:32:01\n", - " \n", - " Pass\n", - " imported\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " Almonds\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 3\n", - " 01C6C5BE-CEA6-54AF-3924-B0BAD69335\n", - " (73)9335\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", - " 1\n", - " Prox01xk(73)1\n", - " Ash\n", - " 0.69\n", - " % total weight\n", - " 2024-10-03 10:31:01\n", - " \n", - " Pass\n", - " import failed\n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " 4\n", + " Walnuts\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 4\n", - " 126745C7-DD41-2F6D-0DC5-28DBCA415F\n", - " (73)415F\n", - " Oakleaf\n", - " Oak-TmPm01A(73)\n", - " Tomato pomace\n", - " As Is\n", - " 4C\n", - " Prox01xk\n", - " 2\n", - " Prox01xk(73)2\n", - " Ash\n", - " 0.89\n", - " % total weight\n", - " 2024-10-03 10:31:31\n", - " \n", - " Pass\n", - " \n", - " \n", - " Proximate analysis\n", - " xkang2@lbl.gov\n", + " 5\n", + " Sweet potatoes\n", + " None\n", + " None\n", + " None\n", " \n", - " \n", - "\n", - "
" - ], - "text/plain": [ - " Prox_UUID_031 Record_ID Source_codename \\\n", - "0 D7965110-407F-E356-D41D-B3B9A2B7B7 (73)B7B7 Oakleaf \n", - "1 C8FEA984-2E9A-8DEF-55FB-1A9D7D9BA8 (73)9BA8 Oakleaf \n", - "2 DF304D5D-3A85-4881-7142-6D4E5F957D (73)957D Oakleaf \n", - "3 01C6C5BE-CEA6-54AF-3924-B0BAD69335 (73)9335 Oakleaf \n", - "4 126745C7-DD41-2F6D-0DC5-28DBCA415F (73)415F Oakleaf \n", - "\n", - " Prepared_sample Resource Preparation_method Storage_cond \\\n", - "0 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "1 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "2 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "3 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "4 Oak-TmPm01A(73) Tomato pomace As Is 4C \n", - "\n", - " Exper_abbrev Repl_no Repl_ID Parameter Value Unit \\\n", - "0 Prox01xk 1 Prox01xk(73)1 Moisture 61.85 % total weight \n", - "1 Prox01xk 2 Prox01xk(73)2 Moisture 63.21 % total weight \n", - "2 Prox01xk 3 Prox01xk(73)3 Moisture 63.27 % total weight \n", - "3 Prox01xk 1 Prox01xk(73)1 Ash 0.69 % total weight \n", - "4 Prox01xk 2 Prox01xk(73)2 Ash 0.89 % total weight \n", - "\n", - " Created_at Updated_at QC_result Upload_status Note \\\n", - "0 2024-10-02 10:31:01 Pass not ready \n", - "1 2024-10-02 10:31:31 Pass ready \n", - "2 2024-10-02 10:32:01 Pass imported \n", - "3 2024-10-03 10:31:01 Pass import failed \n", - "4 2024-10-03 10:31:31 Pass \n", - "\n", - " Analysis_type Analyst_email \n", - "0 Proximate analysis xkang2@lbl.gov \n", - "1 Proximate analysis xkang2@lbl.gov \n", - "2 Proximate analysis xkang2@lbl.gov \n", - "3 Proximate analysis xkang2@lbl.gov \n", - "4 Proximate analysis xkang2@lbl.gov " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:44.272 | INFO    | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:44.272 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.593 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.593 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.595 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.595 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted Ultimate data.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Ult_UUID_037", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Ult_sample_name", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, + " \n", + " 5\n", + " 6\n", + " Algae\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 6\n", + " 7\n", + " Olives - processing\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 7\n", + " 8\n", + " Corn - all\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 8\n", + " 9\n", + " Hay - alfalfa\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 9\n", + " 10\n", + " Silage - wheat\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 10\n", + " 11\n", + " Rice\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 11\n", + " 12\n", + " Peaches\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 12\n", + " 13\n", + " Potatoes\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 13\n", + " 14\n", + " Cucumbers and pickles\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 14\n", + " 15\n", + " Pistachios\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 15\n", + " 16\n", + " Cotton\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 16\n", + " 17\n", + " Olives - market\n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + " 17\n", + " 18\n", + " \n", + " None\n", + " None\n", + " None\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " id name note description uri\n", + "0 1 Tomatoes for processing None None None\n", + "1 2 Grapes None None None\n", + "2 3 Almonds None None None\n", + "3 4 Walnuts None None None\n", + "4 5 Sweet potatoes None None None\n", + "5 6 Algae None None None\n", + "6 7 Olives - processing None None None\n", + "7 8 Corn - all None None None\n", + "8 9 Hay - alfalfa None None None\n", + "9 10 Silage - wheat None None None\n", + "10 11 Rice None None None\n", + "11 12 Peaches None None None\n", + "12 13 Potatoes None None None\n", + "13 14 Cucumbers and pickles None None None\n", + "14 15 Pistachios None None None\n", + "15 16 Cotton None None None\n", + "16 17 Olives - market None None None\n", + "17 18 None None None" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sqlmodel import Session, select, create_engine\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "#This module queries the db via the ORM\n", + "\n", + "# Database Connection\n", + "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", + "engine = create_engine(DATABASE_URL)\n", + "print(f\"Connected to database.\")\n", + "\n", + "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", + "\n", + "#reorders columns so id and name are first\n", + "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", + "\n", + "primary_ag_product = primary_ag_product[[*cols]]\n", + "\n", + "primary_ag_product\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "cmp_uuid_033", + "rawType": "object", + "type": "string" + }, + { + "name": "record_id", + "rawType": "object", + "type": "string" + }, + { + "name": "prepared_sample", + "rawType": "object", + "type": "string" + }, + { + "name": "resource", + "rawType": "object", + "type": "string" + }, { - "name": "Parameter", + "name": "preparation_method", + "rawType": "object", + "type": "string" + }, + { + "name": "storage_cond", "rawType": "object", "type": "string" }, { - "name": "Value", + "name": "exper_abbrev", "rawType": "object", "type": "string" }, { - "name": "Unit", + "name": "repl_no", + "rawType": "int64", + "type": "integer" + }, + { + "name": "repl_id", "rawType": "object", "type": "string" }, { - "name": "Created_at", + "name": "parameter", "rawType": "object", "type": "string" }, { - "name": "Updated_at", + "name": "value", + "rawType": "float64", + "type": "float" + }, + { + "name": "unit", "rawType": "object", "type": "string" }, { - "name": "QC_result", + "name": "created_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "updated_at", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "qc_result", "rawType": "object", "type": "string" }, { - "name": "Note", + "name": "note", "rawType": "object", "type": "string" }, { - "name": "Analysis_type", + "name": "analysis_type", "rawType": "object", "type": "string" }, { - "name": "Equipment", + "name": "equipment", "rawType": "object", "type": "string" }, { - "name": "Raw_data_URL", + "name": "raw_data_url", "rawType": "object", "type": "string" }, { - "name": "Analyst_email", + "name": "analyst_email", "rawType": "object", "type": "string" }, { - "name": "Upload_status", + "name": "upload_status", "rawType": "object", "type": "string" } ], - "ref": "61714e59-0541-4b74-9d38-a37794cbd34d", - "rows": [ - [ - "0", - "421617A5-E50E-9642-2974-B3275FE822", - ")U22E822", - "Hum-AlmHu023KM2(15)U22", - "Hum-AlmHu023KM2(15)", - "Almond Hulls", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(15)1", - "DM", - "8.86E+01", - "pc", - "", - "", - "Pass", - "1", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "1", - "7E7919C2-5DB4-6BEF-75E2-7E51321200", - ")U001200", - "Hum-AlmHu023KM2(15)U00", - "Hum-AlmHu023KM2(15)", - "Almond Hulls", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(15)1", - "DM", - "", - "pc", - "", - "", - "Fail", - " 1 dup", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "2", - "3AA85881-1185-642F-C44B-41AD2275D2", - ")UD275D2", - "Hum-AlmSh022KM2(13)UD2", - "Hum-AlmSh022KM2(13)", - "Almond Shells", - "Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(13)1", - "DM", - "8.13E+01", - "pc", - "", - "", - "Pass", - "2", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "3", - "FA418804-6C4F-4C90-D78F-84D7DF54D3", - ")UD354D3", - "Ene-WaSh017OKM2(82)UD3", - "Ene-WaSh017OKM2(82)", - "Walnut Shells", - "Oven Dry + Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(82)1", - "DM", - "9.22E+01", - "pc", - "", - "", - "Pass", - "3", - "Ultimate analysis", - "", - "", - "", - "" - ], - [ - "4", - "6FDACBFC-7E0B-473B-444F-85B7650267", - ")U670267", - "Ebo-GpPm010OKM2(1B)U67", - "Ebo-GpPm010OKM2(1B)", - "Grape pomace", - "Oven Dry + Knife Mill (2mm)", - "RT vacuum sealed", - "Ult26kh", - "1", - "Ult26kh(1B)1", - "DM", - "9.38E+01", - "pc", - "", - "", - "Pass", - "4", - "Ultimate analysis", - "", - "", - "", - "" - ] - ], - "shape": { - "columns": 22, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Ult_UUID_037Record_IDUlt_sample_namePrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_ID...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
0421617A5-E50E-9642-2974-B3275FE822)U22E822Hum-AlmHu023KM2(15)U22Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcPass1Ultimate analysis
17E7919C2-5DB4-6BEF-75E2-7E51321200)U001200Hum-AlmHu023KM2(15)U00Hum-AlmHu023KM2(15)Almond HullsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(15)1...pcFail1 dupUltimate analysis
23AA85881-1185-642F-C44B-41AD2275D2)UD275D2Hum-AlmSh022KM2(13)UD2Hum-AlmSh022KM2(13)Almond ShellsKnife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(13)1...pcPass2Ultimate analysis
3FA418804-6C4F-4C90-D78F-84D7DF54D3)UD354D3Ene-WaSh017OKM2(82)UD3Ene-WaSh017OKM2(82)Walnut ShellsOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(82)1...pcPass3Ultimate analysis
46FDACBFC-7E0B-473B-444F-85B7650267)U670267Ebo-GpPm010OKM2(1B)U67Ebo-GpPm010OKM2(1B)Grape pomaceOven Dry + Knife Mill (2mm)RT vacuum sealedUlt26kh1Ult26kh(1B)1...pcPass4Ultimate analysis
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " Ult_UUID_037 Record_ID Ult_sample_name \\\n", - "0 421617A5-E50E-9642-2974-B3275FE822 )U22E822 Hum-AlmHu023KM2(15)U22 \n", - "1 7E7919C2-5DB4-6BEF-75E2-7E51321200 )U001200 Hum-AlmHu023KM2(15)U00 \n", - "2 3AA85881-1185-642F-C44B-41AD2275D2 )UD275D2 Hum-AlmSh022KM2(13)UD2 \n", - "3 FA418804-6C4F-4C90-D78F-84D7DF54D3 )UD354D3 Ene-WaSh017OKM2(82)UD3 \n", - "4 6FDACBFC-7E0B-473B-444F-85B7650267 )U670267 Ebo-GpPm010OKM2(1B)U67 \n", - "\n", - " Prepared_sample Resource Preparation_method \\\n", - "0 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", - "1 Hum-AlmHu023KM2(15) Almond Hulls Knife Mill (2mm) \n", - "2 Hum-AlmSh022KM2(13) Almond Shells Knife Mill (2mm) \n", - "3 Ene-WaSh017OKM2(82) Walnut Shells Oven Dry + Knife Mill (2mm) \n", - "4 Ebo-GpPm010OKM2(1B) Grape pomace Oven Dry + Knife Mill (2mm) \n", - "\n", - " Storage_cond Exper_abbrev Repl_no Repl_ID ... Unit Created_at \\\n", - "0 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", - "1 RT vacuum sealed Ult26kh 1 Ult26kh(15)1 ... pc \n", - "2 RT vacuum sealed Ult26kh 1 Ult26kh(13)1 ... pc \n", - "3 RT vacuum sealed Ult26kh 1 Ult26kh(82)1 ... pc \n", - "4 RT vacuum sealed Ult26kh 1 Ult26kh(1B)1 ... pc \n", - "\n", - " Updated_at QC_result Note Analysis_type Equipment Raw_data_URL \\\n", - "0 Pass 1 Ultimate analysis \n", - "1 Fail 1 dup Ultimate analysis \n", - "2 Pass 2 Ultimate analysis \n", - "3 Pass 3 Ultimate analysis \n", - "4 Pass 4 Ultimate analysis \n", - "\n", - " Analyst_email Upload_status \n", - "0 \n", - "1 \n", - "2 \n", - "3 \n", - "4 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:45.631 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "20:54:45.631 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:47.211 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "20:54:47.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
20:54:47.214 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "20:54:47.214 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Successfully extracted CmpAna data.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Cmp_UUID_033", - "rawType": "object", - "type": "string" - }, - { - "name": "Record_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "Resource", - "rawType": "object", - "type": "string" - }, - { - "name": "Preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "Storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "Exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_no", - "rawType": "object", - "type": "string" - }, - { - "name": "Repl_ID", - "rawType": "object", - "type": "string" - }, - { - "name": "Parameter", - "rawType": "object", - "type": "string" - }, - { - "name": "Value", - "rawType": "object", - "type": "string" - }, - { - "name": "Unit", - "rawType": "object", - "type": "string" - }, - { - "name": "Created_at", - "rawType": "object", - "type": "string" - }, - { - "name": "Updated_at", - "rawType": "object", - "type": "string" - }, - { - "name": "QC_result", - "rawType": "object", - "type": "string" - }, - { - "name": "Note", - "rawType": "object", - "type": "string" - }, - { - "name": "Analysis_type", - "rawType": "object", - "type": "string" - }, - { - "name": "Equipment", - "rawType": "object", - "type": "string" - }, - { - "name": "Raw_data_URL", - "rawType": "object", - "type": "string" - }, - { - "name": "Analyst_email", - "rawType": "object", - "type": "string" - }, - { - "name": "Upload_status", - "rawType": "object", - "type": "string" - } - ], - "ref": "2dd03f9d-30e4-4e23-af07-409bfdca5047", - "rows": [ - [ - "0", - "3EE2993D-86E3-1F16-C7EA-F8D555E114", - "(85)E114", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "Glucan", - "14.16", - "% dry weight", - "1/23/2025 9:00:01", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "1", - "46878EF9-1226-22A0-D5D8-CF65E241CB", - "(85)41CB", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "Glucan", - "14.18", - "% dry weight", - "1/23/2025 9:00:16", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "2", - "76A7A2F4-C4E4-E60F-1187-DEC6E02246", - "(85)2246", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "Glucan", - "14.12", - "% dry weight", - "1/23/2025 9:00:31", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "3", - "7A136832-286B-07CB-62DE-ACF52F9311", - "(85)9311", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "Glucose", - "15.74", - "% dry weight", - "1/23/2025 9:00:46", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "4", - "B709ECEE-F9A6-A55D-A59E-93B7B863D7", - "(85)63D7", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "Glucose", - "15.75", - "% dry weight", - "1/23/2025 9:01:01", - "", - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ] - ], - "shape": { - "columns": 21, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Cmp_UUID_033Record_IDPrepared_sampleResourcePreparation_methodStorage_condExper_abbrevRepl_noRepl_IDParameter...UnitCreated_atUpdated_atQC_resultNoteAnalysis_typeEquipmentRaw_data_URLAnalyst_emailUpload_status
03EE2993D-86E3-1F16-C7EA-F8D555E114(85)E114Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucan...% dry weight1/23/2025 9:00:01passChemical compositionxkang2@lbl.govready
146878EF9-1226-22A0-D5D8-CF65E241CB(85)41CBOak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucan...% dry weight1/23/2025 9:00:16passChemical compositionxkang2@lbl.govready
276A7A2F4-C4E4-E60F-1187-DEC6E02246(85)2246Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk3Cmp04xk(85)3Glucan...% dry weight1/23/2025 9:00:31passChemical compositionxkang2@lbl.govready
37A136832-286B-07CB-62DE-ACF52F9311(85)9311Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk1Cmp04xk(85)1Glucose...% dry weight1/23/2025 9:00:46passChemical compositionxkang2@lbl.govready
4B709ECEE-F9A6-A55D-A59E-93B7B863D7(85)63D7Oak-TmPm01O(85)Tomato pomaceOven dryRT vacuum sealedCmp04xk2Cmp04xk(85)2Glucose...% dry weight1/23/2025 9:01:01passChemical compositionxkang2@lbl.govready
\n", - "

5 rows × 21 columns

\n", - "
" - ], - "text/plain": [ - " Cmp_UUID_033 Record_ID Prepared_sample \\\n", - "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", - "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", - "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", - "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", - "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", - "\n", - " Resource Preparation_method Storage_cond Exper_abbrev Repl_no \\\n", - "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", - "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", - "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 3 \n", - "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 1 \n", - "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk 2 \n", - "\n", - " Repl_ID Parameter ... Unit Created_at Updated_at \\\n", - "0 Cmp04xk(85)1 Glucan ... % dry weight 1/23/2025 9:00:01 \n", - "1 Cmp04xk(85)2 Glucan ... % dry weight 1/23/2025 9:00:16 \n", - "2 Cmp04xk(85)3 Glucan ... % dry weight 1/23/2025 9:00:31 \n", - "3 Cmp04xk(85)1 Glucose ... % dry weight 1/23/2025 9:00:46 \n", - "4 Cmp04xk(85)2 Glucose ... % dry weight 1/23/2025 9:01:01 \n", - "\n", - " QC_result Note Analysis_type Equipment Raw_data_URL Analyst_email \\\n", - "0 pass Chemical composition xkang2@lbl.gov \n", - "1 pass Chemical composition xkang2@lbl.gov \n", - "2 pass Chemical composition xkang2@lbl.gov \n", - "3 pass Chemical composition xkang2@lbl.gov \n", - "4 pass Chemical composition xkang2@lbl.gov \n", - "\n", - " Upload_status \n", - "0 ready \n", - "1 ready \n", - "2 ready \n", - "3 ready \n", - "4 ready \n", - "\n", - "[5 rows x 21 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Cmp Ana extraction process completed.\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import pandas as pd\n", - "import numpy as np\n", - "import janitor as jn\n", - "from IPython.display import display\n", - "\n", - "# --- Robustly find the project root ---\n", - "# The project root is the directory containing the 'pixi.toml' file.\n", - "path = os.getcwd()\n", - "project_root = None\n", - "while path != os.path.dirname(path): # Stop at the filesystem root\n", - " if 'pixi.toml' in os.listdir(path):\n", - " project_root = path\n", - " break\n", - " path = os.path.dirname(path)\n", - "\n", - "if not project_root:\n", - " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", - "\n", - "# --- Add project root to sys.path ---\n", - "if project_root not in sys.path:\n", - " sys.path.insert(0, project_root)\n", - " print(f\"Added project root '{project_root}' to sys.path\")\n", - "else:\n", - " print(f\"Project root '{project_root}' is already in sys.path\")\n", - "\n", - "# --- Import the module ---\n", - "try:\n", - " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", - " print(\"Successfully imported all module.\")\n", - "except ImportError as e:\n", - " print(f\"Failed to import modules: {e}\")\n", - " print(f\"\\nFull sys.path: {sys.path}\")\n", - "\n", - "# --- Run the extraction ---\n", - "if 'proximate' in locals():\n", - " try:\n", - " # Pass the project_root to the extract function\n", - " df = proximate.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted proximate data.\")\n", - " display(df.head())\n", - " else:\n", - " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", - "\n", - "if 'ultimate' in locals():\n", - " try:\n", - " df2 = ultimate.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted Ultimate data.\")\n", - " display(df2.head())\n", - " else:\n", - " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during extraction: {e}\")\n", - "\n", - "if 'cmpana' in locals():\n", - " try:\n", - " df3 = cmpana.extract(project_root=project_root)\n", - " if df is not None:\n", - " print(\"\\nSuccessfully extracted CmpAna data.\")\n", - " display(df3.head())\n", - " else:\n", - " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", - " except Exception as e:\n", - " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", - " finally:\n", - " print(\"\\nCmp Ana extraction process completed.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def clean_the_gsheets(df):\n", - " # 1. Clean names and drop rows\n", - " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", - "\n", - " # 2. Coerce types (using errors='coerce' handles messy string data)\n", - " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", - " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", - "\n", - " # 3. Dates\n", - " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", - " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", - "\n", - " # 4. Convert remaining objects to best possible types (like strings)\n", - " df = df.convert_dtypes()\n", - " \n", - " return df # Return the FULL dataframe, not just .head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframes = [df, df2, df3]\n", - "\n", - "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", - "\n", - "clean_dataframes[2].head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "summary_stats = clean_dataframes[0].\\\n", - " groupby(['resource', 'parameter'])['value'].\\\n", - " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", - "\n", - "summary_stats" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", - " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", - " agg({'value': 'mean'}).\\\n", - " query('value > 30').\\\n", - " sort_values(by='value', ascending=False).\\\n", - " round({'value': 1})\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", - "\n", - "def is_it_volatile_solids(df):\n", - " df['check'] = \"VS\"\n", - "\n", - " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", - " return df\n", - "\n", - "is_it_volatile_solids(df)\n", - "\n", - "df[['check', 'parameter']]\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", - "def sqrtvalue(df):\n", - " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", - " return df\n", - "\n", - "#List comprehension to apply sqrtvalue to each DataFrame\n", - "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", - "\n", - "# Display the head of the third DataFrame\n", - "clean_rooted_df[2].head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cmpana_raw = cmpana.extract(project_root=project_root)\n", - "\n", - "cmpana_raw.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected to database.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "name", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "unknown" - }, - { - "name": "description", - "rawType": "object", - "type": "unknown" - }, - { - "name": "uri", - "rawType": "object", - "type": "unknown" - } - ], - "ref": "0dabe9e2-728a-486d-9dd1-c1edd094e9c2", - "rows": [ - [ - "0", - "1", - "Tomatoes for processing", - null, - null, - null - ], - [ - "1", - "2", - "Grapes", - null, - null, - null - ], - [ - "2", - "3", - "Almonds", - null, - null, - null - ], - [ - "3", - "4", - "Walnuts", - null, - null, - null - ], - [ - "4", - "5", - "Sweet potatoes", - null, - null, - null - ], - [ - "5", - "6", - "Algae", - null, - null, - null - ], - [ - "6", - "7", - "Olives - processing", - null, - null, - null - ], - [ - "7", - "8", - "Corn - all", - null, - null, - null - ], - [ - "8", - "9", - "Hay - alfalfa", - null, - null, - null - ], - [ - "9", - "10", - "Silage - wheat", - null, - null, - null - ], - [ - "10", - "11", - "Rice", - null, - null, - null - ], - [ - "11", - "12", - "Peaches", - null, - null, - null - ], - [ - "12", - "13", - "Potatoes", - null, - null, - null - ], - [ - "13", - "14", - "Cucumbers and pickles", - null, - null, - null - ], - [ - "14", - "15", - "Pistachios", - null, - null, - null - ], - [ - "15", - "16", - "Cotton", - null, - null, - null - ], - [ - "16", - "17", - "Olives - market", - null, - null, - null - ], - [ - "17", - "18", - "", - null, - null, - null - ] - ], - "shape": { - "columns": 5, - "rows": 18 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamenotedescriptionuri
01Tomatoes for processingNoneNoneNone
12GrapesNoneNoneNone
23AlmondsNoneNoneNone
34WalnutsNoneNoneNone
45Sweet potatoesNoneNoneNone
56AlgaeNoneNoneNone
67Olives - processingNoneNoneNone
78Corn - allNoneNoneNone
89Hay - alfalfaNoneNoneNone
910Silage - wheatNoneNoneNone
1011RiceNoneNoneNone
1112PeachesNoneNoneNone
1213PotatoesNoneNoneNone
1314Cucumbers and picklesNoneNoneNone
1415PistachiosNoneNoneNone
1516CottonNoneNoneNone
1617Olives - marketNoneNoneNone
1718NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " id name note description uri\n", - "0 1 Tomatoes for processing None None None\n", - "1 2 Grapes None None None\n", - "2 3 Almonds None None None\n", - "3 4 Walnuts None None None\n", - "4 5 Sweet potatoes None None None\n", - "5 6 Algae None None None\n", - "6 7 Olives - processing None None None\n", - "7 8 Corn - all None None None\n", - "8 9 Hay - alfalfa None None None\n", - "9 10 Silage - wheat None None None\n", - "10 11 Rice None None None\n", - "11 12 Peaches None None None\n", - "12 13 Potatoes None None None\n", - "13 14 Cucumbers and pickles None None None\n", - "14 15 Pistachios None None None\n", - "15 16 Cotton None None None\n", - "16 17 Olives - market None None None\n", - "17 18 None None None" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sqlmodel import Session, select, create_engine\n", - "import pandas as pd\n", - "import os\n", - "import sys\n", - "\n", - "\n", - "# Database Connection\n", - "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", - "engine = create_engine(DATABASE_URL)\n", - "print(f\"Connected to database.\")\n", - "\n", - "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", - "\n", - "#reorders columns so id and name are first\n", - "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", - "\n", - "primary_ag_product = primary_ag_product[[*cols]]\n", - "\n", - "primary_ag_product\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "description", - "rawType": "object", - "type": "unknown" - }, - { - "name": "id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "name", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "unknown" - }, - { - "name": "uri", - "rawType": "object", - "type": "unknown" - } - ], - "ref": "52090a35-9379-4dc9-b918-583b23513ddd", - "rows": [ - [ - "0", - null, - "1", - "Tomatoes for processing", - null, - null - ], - [ - "1", - null, - "2", - "Grapes", - null, - null - ], - [ - "2", - null, - "3", - "Almonds", - null, - null - ], - [ - "3", - null, - "4", - "Walnuts", - null, - null - ], - [ - "4", - null, - "5", - "Sweet potatoes", - null, - null - ] - ], - "shape": { - "columns": 5, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
descriptionidnamenoteuri
0None1Tomatoes for processingNoneNone
1None2GrapesNoneNone
2None3AlmondsNoneNone
3None4WalnutsNoneNone
4None5Sweet potatoesNoneNone
\n", - "
" - ], - "text/plain": [ - " description id name note uri\n", - "0 None 1 Tomatoes for processing None None\n", - "1 None 2 Grapes None None\n", - "2 None 3 Almonds None None\n", - "3 None 4 Walnuts None None\n", - "4 None 5 Sweet potatoes None None" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sqlalchemy.orm import Session\n", - "from sqlalchemy import select\n", - "import pandas as pd\n", - "import os\n", - "import sys\n", - "\n", - "# --- project root discovery (unchanged) ---\n", - "path = os.getcwd()\n", - "project_root = None\n", - "while path != os.path.dirname(path):\n", - " if 'pixi.toml' in os.listdir(path):\n", - " project_root = path\n", - " break\n", - " path = os.path.dirname(path)\n", - "\n", - "if not project_root:\n", - " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", - "\n", - "if project_root not in sys.path:\n", - " sys.path.insert(0, project_root)\n", - "\n", - "# --- imports ---\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", - "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", - "\n", - "# --- query + dataframe ---\n", - "with Session(engine) as db:\n", - " stmt = select(*PrimaryAgProduct.__table__.columns)\n", - " rows = db.execute(stmt).mappings().all()\n", - "\n", - "df = pd.DataFrame(rows)\n", - "\n", - "df.head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
21:07:54.222 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
-       "
\n" - ], - "text/plain": [ - "21:07:54.222 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
21:07:59.598 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
-       "
\n" - ], - "text/plain": [ - "21:07:59.598 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
21:07:59.601 | INFO    | Task run 'extract' - Finished in state Completed()\n",
-       "
\n" - ], - "text/plain": [ - "21:07:59.601 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from sqlalchemy.orm import Session\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", - "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", - "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", - " replace_name_with_id_df,\n", - ") \n", - "\n", - "#This extractst the raw proximate data\n", - "df = cmpana.extract(project_root=project_root)\n", - "\n", - "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", - "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", - "\n", - "#this replaces the names with IDs\n", - "with Session(engine) as db:\n", - " parameter_ids = replace_name_with_id_df(\n", - " db=db,\n", - " df=test_df,\n", - " ref_model=Parameter,\n", - " name_column_name=\"name\", # column in df + table\n", - " id_column_name=\"id\", # PK column in table\n", - " final_column_name=\"parameter_id\"\n", - " )\n", - "\n", - "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", - "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", - "\n", - "#resource_id_mapping\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "cmp_uuid_033", - "rawType": "string", - "type": "string" - }, - { - "name": "record_id", - "rawType": "string", - "type": "string" - }, - { - "name": "prepared_sample", - "rawType": "string", - "type": "string" - }, - { - "name": "resource", - "rawType": "string", - "type": "string" - }, - { - "name": "preparation_method", - "rawType": "string", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "string", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "string", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "Int32", - "type": "integer" - }, - { - "name": "repl_id", - "rawType": "string", - "type": "string" - }, - { - "name": "value", - "rawType": "Float32", - "type": "float" - }, - { - "name": "unit", - "rawType": "string", - "type": "string" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "string", - "type": "string" - }, - { - "name": "note", - "rawType": "string", - "type": "string" - }, - { - "name": "analysis_type", - "rawType": "string", - "type": "string" - }, - { - "name": "equipment", - "rawType": "string", - "type": "string" - }, - { - "name": "raw_data_url", - "rawType": "string", - "type": "string" - }, - { - "name": "analyst_email", - "rawType": "string", - "type": "string" - }, - { - "name": "upload_status", - "rawType": "string", - "type": "string" - }, - { - "name": "parameter_id", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "91fad99c-9137-48ce-af0f-4004b3241e88", - "rows": [ - [ - "0", - "3EE2993D-86E3-1F16-C7EA-F8D555E114", - "(85)E114", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "14.16", - "% dry weight", - "2025-01-23 09:00:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "1", - "46878EF9-1226-22A0-D5D8-CF65E241CB", - "(85)41CB", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "14.18", - "% dry weight", - "2025-01-23 09:00:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "2", - "76A7A2F4-C4E4-E60F-1187-DEC6E02246", - "(85)2246", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "14.12", - "% dry weight", - "2025-01-23 09:00:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "3", - "7A136832-286B-07CB-62DE-ACF52F9311", - "(85)9311", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "15.74", - "% dry weight", - "2025-01-23 09:00:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "4", - "B709ECEE-F9A6-A55D-A59E-93B7B863D7", - "(85)63D7", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "15.75", - "% dry weight", - "2025-01-23 09:01:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "5", - "35080403-BB4B-776B-E045-4627C307BD", - "(85)07BD", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "15.68", - "% dry weight", - "2025-01-23 09:01:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "6", - "D83F3328-931D-582E-B563-4DFF118C05", - "(85)8C05", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "9.88", - "% dry weight", - "2025-01-23 09:01:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "7", - "B3F6AA18-E51A-2585-8417-113DE0F026", - "(85)F026", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "9.25", - "% dry weight", - "2025-01-23 09:01:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "8", - "094CFB14-0D19-C54E-51C4-6BA548A6D4", - "(85)A6D4", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "9.43", - "% dry weight", - "2025-01-23 09:02:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "9", - "99D17377-62B8-2124-5391-62BC1F4630", - "(85)4630", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "11.22", - "% dry weight", - "2025-01-23 09:02:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "10", - "9B784971-B92C-A870-DA69-C4D771B716", - "(85)B716", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "10.51", - "% dry weight", - "2025-01-23 09:02:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "11", - "57C508A2-7489-2AFB-23B2-1630C38BD5", - "(85)8BD5", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "10.72", - "% dry weight", - "2025-01-23 09:02:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "12", - "40EAE6DB-4CD6-3574-949F-70B03A51C8", - "(85)51C8", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(85)1", - "39.54", - "% dry weight", - "2025-01-23 09:03:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "13", - "A752F059-ECF4-FA5B-A9D2-7DF8963329", - "(85)3329", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(85)2", - "39.9", - "% dry weight", - "2025-01-23 09:03:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "14", - "BEB4369F-D40F-DF6A-30F8-75B91C5B1B", - "(85)5B1B", - "Oak-TmPm01O(85)", - "Tomato pomace", - "Oven dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(85)3", - "40.88", - "% dry weight", - "2025-01-23 09:03:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "15", - "00CAC475-8E18-2318-AA6A-2AE068CADF", - "(D2)CADF", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "19.04", - "% dry weight", - "2025-01-23 09:03:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "16", - "2DA01B52-D301-53C9-220D-A51216BDB6", - "(D2)BDB6", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "18.9", - "% dry weight", - "2025-01-23 09:04:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "17", - "F8D1C324-F3F5-7754-494E-7BBFB262D0", - "(D2)62D0", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "18.98", - "% dry weight", - "2025-01-23 09:04:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "18", - "84FE8C5C-5D37-B0EA-6488-B883CEA7F3", - "(D2)A7F3", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "21.15", - "% dry weight", - "2025-01-23 09:04:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "19", - "E705EAF1-69EB-ADB3-620B-1DD93F4D07", - "(D2)4D07", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "21.0", - "% dry weight", - "2025-01-23 09:04:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "20", - "F6A97D04-E898-06F1-A90B-0EE254D758", - "(D2)D758", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "21.08", - "% dry weight", - "2025-01-23 09:05:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "21", - "6948E00A-4858-6F19-23BC-AA016F12CB", - "(D2)12CB", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "8.84", - "% dry weight", - "2025-01-23 09:05:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "22", - "EEC9EA54-3C1A-6154-B953-401C89CEA7", - "(D2)CEA7", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "8.46", - "% dry weight", - "2025-01-23 09:05:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "23", - "62E4A908-ED27-590F-0DD2-EA254F1E4C", - "(D2)1E4C", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "8.72", - "% dry weight", - "2025-01-23 09:05:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "24", - "AA2BBE0D-FD98-98F9-9880-CDB56142A2", - "(D2)42A2", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "10.04", - "% dry weight", - "2025-01-23 09:06:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "25", - "5186F84F-8BBB-98A7-8FF9-CF85CCD344", - "(D2)D344", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "9.62", - "% dry weight", - "2025-01-23 09:06:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "26", - "2BAF4360-89A7-C605-2141-827901DA56", - "(D2)DA56", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "9.91", - "% dry weight", - "2025-01-23 09:06:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "27", - "B9D049D7-363C-365F-3750-331CEC2FAE", - "(D2)2FAE", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(D2)1", - "30.53", - "% dry weight", - "2025-01-23 09:06:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "28", - "4E056928-76F1-99E1-4F78-FD31CC0B71", - "(D2)0B71", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(D2)2", - "31.57", - "% dry weight", - "2025-01-23 09:07:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "29", - "4734CD60-9A40-8ECD-7E0D-FA2AD6D96D", - "(D2)D96D", - "Pin-TmPm02O(D2)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(D2)3", - "31.95", - "% dry weight", - "2025-01-23 09:07:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "30", - "2CD57693-9038-EC9E-EC61-E91793A602", - "(28)A602", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "11.71", - "% dry weight", - "2025-01-23 09:07:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "31", - "BD14399E-2A3B-56A9-AC9C-1689DA75A7", - "(28)75A7", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "13.66", - "% dry weight", - "2025-01-23 09:07:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "32", - "6635AA49-3247-3A08-D00F-CA5447491E", - "(28)491E", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "13.84", - "% dry weight", - "2025-01-23 09:08:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "33", - "EE7D6738-BEE5-87A1-6387-89AE03CFC4", - "(28)CFC4", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "13.01", - "% dry weight", - "2025-01-23 09:08:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "34", - "ACD2BFD6-EA7C-8CBB-F4DA-CAA860F6C9", - "(28)F6C9", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "15.17", - "% dry weight", - "2025-01-23 09:08:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "35", - "1BFB37F6-A5D7-E88E-84A4-DBA38059B5", - "(28)59B5", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "15.37", - "% dry weight", - "2025-01-23 09:08:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "4" - ], - [ - "36", - "D6CD55E5-19B7-5E3A-EEE6-95564F8C33", - "(28)8C33", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(28)1", - "9.36", - "% dry weight", - "2025-01-23 09:09:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "37", - "17DE0AED-2CC5-2B5A-E521-049F78F0FE", - "(28)F0FE", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "11.26", - "% dry weight", - "2025-01-23 09:09:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], - [ - "38", - "4D593E32-1FDA-373B-EE45-38FB01847B", - "(28)847B", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "11.06", - "% dry weight", - "2025-01-23 09:09:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "2" - ], + "ref": "b4e5431c-6847-48e5-894f-3a2387854a2b", + "rows": [ [ - "39", - "84904964-58B9-A45E-3F79-6747BF7919", - "(28)7919", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "0", + "3ee2993d-86e3-1f16-c7ea-f8d555e114", + "(85)e114", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "1", - "Cmp04xk(28)1", - "10.64", - "% dry weight", - "2025-01-23 09:09:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "40", - "808F6FAC-F9A6-A9EB-56D5-EECA3BA3D2", - "(28)A3D2", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "2", - "Cmp04xk(28)2", - "12.8", - "% dry weight", - "2025-01-23 09:10:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "5" - ], - [ - "41", - "67FD9F7C-8CE8-04F4-2CB5-6C42532429", - "(28)2429", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "12.56", + "cmp04xk(85)1", + "glucan", + "14.15999984741211", "% dry weight", - "2025-01-23 09:10:16", + "2025-01-23 09:00:01", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "5" + "ready" ], [ - "42", - "90E8067F-045F-3E6E-5BAA-94BEE9D0CF", - "(28)D0CF", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", "1", - "Cmp04xk(28)1", - "40.12", - "% dry weight", - "2025-01-23 09:10:31", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "43", - "112546A5-714C-B29F-808B-CBDE6DE48E", - "(28)E48E", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "46878ef9-1226-22a0-d5d8-cf65e241cb", + "(85)41cb", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "2", - "Cmp04xk(28)2", - "39.76", - "% dry weight", - "2025-01-23 09:10:46", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "44", - "2FF25721-ED5B-05E4-2DC5-5799261266", - "(28)1266", - "Riv-TmPm03O(28)", - "Tomato pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "3", - "Cmp04xk(28)3", - "40.34", - "% dry weight", - "2025-01-23 09:11:01", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "1" - ], - [ - "45", - "7BC7134E-F8BB-F153-DC50-67A1161509", - "(AB)1509", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", - "1", - "Cmp04xk(AB)1", - "7.95", + "cmp04xk(85)2", + "glucan", + "14.180000305175781", "% dry weight", - "2025-01-22 09:00:01", + "2025-01-23 09:00:16", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "6" + "ready" ], [ - "46", - "F442FF50-46CF-317D-9708-14F6461273", - "(AB)1273", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", "2", - "Cmp04xk(AB)2", - "7.77", - "% dry weight", - "2025-01-22 09:00:16", - null, - "pass", - "", - "Chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready", - "6" - ], - [ - "47", - "C80175E1-8D21-920C-B982-A246941FCB", - "(AB)1FCB", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "76a7a2f4-c4e4-e60f-1187-dec6e02246", + "(85)2246", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "3", - "Cmp04xk(AB)3", - "7.96", + "cmp04xk(85)3", + "glucan", + "14.119999885559082", "% dry weight", - "2025-01-22 09:00:31", + "2025-01-23 09:00:31", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "6" + "ready" ], [ - "48", - "63CF426B-99C5-3896-C472-49BBAE19CD", - "(AB)19CD", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "3", + "7a136832-286b-07cb-62de-acf52f9311", + "(85)9311", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "1", - "Cmp04xk(AB)1", - "8.84", + "cmp04xk(85)1", + "glucose", + "15.739999771118164", "% dry weight", - "2025-01-22 09:00:46", + "2025-01-23 09:00:46", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "4" + "ready" ], [ - "49", - "A9D902B0-033D-29B4-C477-1A7D3E57F4", - "(AB)57F4", - "Map-GpPm04O(AB)", - "Grape pomace", - "Oven Dry", - "RT vacuum sealed", - "Cmp04xk", + "4", + "b709ecee-f9a6-a55d-a59e-93b7b863d7", + "(85)63d7", + "oak-tmpm01o(85)", + "tomato pomace", + "oven dry", + "rt vacuum sealed", + "cmp04xk", "2", - "Cmp04xk(AB)2", - "8.64", + "cmp04xk(85)2", + "glucose", + "15.75", "% dry weight", - "2025-01-22 09:01:01", + "2025-01-23 09:01:01", null, "pass", "", - "Chemical composition", + "chemical composition", "", "", "xkang2@lbl.gov", - "ready", - "4" + "ready" ] ], "shape": { "columns": 21, - "rows": 390 + "rows": 5 } }, "text/html": [ @@ -3772,8 +885,9 @@ " exper_abbrev\n", " repl_no\n", " repl_id\n", - " value\n", + " parameter\n", " ...\n", + " unit\n", " created_at\n", " updated_at\n", " qc_result\n", @@ -3783,353 +897,513 @@ " raw_data_url\n", " analyst_email\n", " upload_status\n", - " parameter_id\n", " \n", " \n", " \n", " \n", " 0\n", - " 3EE2993D-86E3-1F16-C7EA-F8D555E114\n", - " (85)E114\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " 3ee2993d-86e3-1f16-c7ea-f8d555e114\n", + " (85)e114\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 1\n", - " Cmp04xk(85)1\n", - " 14.16\n", + " cmp04xk(85)1\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:01\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 1\n", - " 46878EF9-1226-22A0-D5D8-CF65E241CB\n", - " (85)41CB\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " 46878ef9-1226-22a0-d5d8-cf65e241cb\n", + " (85)41cb\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 2\n", - " Cmp04xk(85)2\n", - " 14.18\n", + " cmp04xk(85)2\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:16\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 2\n", - " 76A7A2F4-C4E4-E60F-1187-DEC6E02246\n", + " 76a7a2f4-c4e4-e60f-1187-dec6e02246\n", " (85)2246\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 3\n", - " Cmp04xk(85)3\n", - " 14.12\n", + " cmp04xk(85)3\n", + " glucan\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:31\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 6\n", " \n", " \n", " 3\n", - " 7A136832-286B-07CB-62DE-ACF52F9311\n", + " 7a136832-286b-07cb-62de-acf52f9311\n", " (85)9311\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 1\n", - " Cmp04xk(85)1\n", - " 15.74\n", + " cmp04xk(85)1\n", + " glucose\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:00:46\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 4\n", " \n", " \n", " 4\n", - " B709ECEE-F9A6-A55D-A59E-93B7B863D7\n", - " (85)63D7\n", - " Oak-TmPm01O(85)\n", - " Tomato pomace\n", - " Oven dry\n", - " RT vacuum sealed\n", - " Cmp04xk\n", + " b709ecee-f9a6-a55d-a59e-93b7b863d7\n", + " (85)63d7\n", + " oak-tmpm01o(85)\n", + " tomato pomace\n", + " oven dry\n", + " rt vacuum sealed\n", + " cmp04xk\n", " 2\n", - " Cmp04xk(85)2\n", - " 15.75\n", + " cmp04xk(85)2\n", + " glucose\n", " ...\n", + " % dry weight\n", " 2025-01-23 09:01:01\n", " NaT\n", " pass\n", " \n", - " Chemical composition\n", + " chemical composition\n", " \n", " \n", " xkang2@lbl.gov\n", " ready\n", - " 4\n", " \n", + " \n", + "\n", + "

5 rows × 21 columns

\n", + "" + ], + "text/plain": [ + " cmp_uuid_033 record_id prepared_sample \\\n", + "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 oak-tmpm01o(85) \n", + "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb oak-tmpm01o(85) \n", + "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 oak-tmpm01o(85) \n", + "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 oak-tmpm01o(85) \n", + "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 oak-tmpm01o(85) \n", + "\n", + " resource preparation_method storage_cond exper_abbrev repl_no \\\n", + "0 tomato pomace oven dry rt vacuum sealed cmp04xk 1 \n", + "1 tomato pomace oven dry rt vacuum sealed cmp04xk 2 \n", + "2 tomato pomace oven dry rt vacuum sealed cmp04xk 3 \n", + "3 tomato pomace oven dry rt vacuum sealed cmp04xk 1 \n", + "4 tomato pomace oven dry rt vacuum sealed cmp04xk 2 \n", + "\n", + " repl_id parameter ... unit created_at updated_at \\\n", + "0 cmp04xk(85)1 glucan ... % dry weight 2025-01-23 09:00:01 NaT \n", + "1 cmp04xk(85)2 glucan ... % dry weight 2025-01-23 09:00:16 NaT \n", + "2 cmp04xk(85)3 glucan ... % dry weight 2025-01-23 09:00:31 NaT \n", + "3 cmp04xk(85)1 glucose ... % dry weight 2025-01-23 09:00:46 NaT \n", + "4 cmp04xk(85)2 glucose ... % dry weight 2025-01-23 09:01:01 NaT \n", + "\n", + " qc_result note analysis_type equipment raw_data_url analyst_email \\\n", + "0 pass chemical composition xkang2@lbl.gov \n", + "1 pass chemical composition xkang2@lbl.gov \n", + "2 pass chemical composition xkang2@lbl.gov \n", + "3 pass chemical composition xkang2@lbl.gov \n", + "4 pass chemical composition xkang2@lbl.gov \n", + "\n", + " upload_status \n", + "0 ready \n", + "1 ready \n", + "2 ready \n", + "3 ready \n", + "4 ready \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataframes = [df, df2, df3]\n", + "\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", + "\n", + "clean_dataframes[2].head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "description", + "rawType": "object", + "type": "unknown" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "name", + "rawType": "object", + "type": "string" + }, + { + "name": "note", + "rawType": "object", + "type": "unknown" + }, + { + "name": "uri", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "bed16ed0-25aa-43f6-b88b-bace074090d9", + "rows": [ + [ + "0", + null, + "1", + "Tomatoes for processing", + null, + null + ], + [ + "1", + null, + "2", + "Grapes", + null, + null + ], + [ + "2", + null, + "3", + "Almonds", + null, + null + ], + [ + "3", + null, + "4", + "Walnuts", + null, + null + ], + [ + "4", + null, + "5", + "Sweet potatoes", + null, + null + ] + ], + "shape": { + "columns": 5, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
descriptionidnamenoteuri
..................................................................0None1Tomatoes for processingNoneNone
38579799601-0E25-3832-3BBF-F4BD9EE973(A0)E973Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1None2Cmp17xk(A0)217.290001...2025-03-04 09:55:00NaTpassChemical compositionxkang2@lbl.govnot ready5
386A4E3393F-72C3-A0ED-C041-105A33031A(A0)031AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)317.5...2025-03-04 09:55:15NaTpassChemical compositionxkang2@lbl.govnot ready5GrapesNoneNone
3873D5A85F8-31A2-0CC2-74E6-05D8197C3A(A0)7C3AHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk1Cmp17xk(A0)135.970001...2025-03-04 09:55:30NaTpassChemical compositionxkang2@lbl.govnot ready2None3AlmondsNoneNone
388C7AE456E-4DCD-AE34-A0D0-BD39D04E42(A0)4E42Hum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk2Cmp17xk(A0)235.279999...2025-03-04 09:55:45NaTpassChemical compositionxkang2@lbl.govnot ready33None4WalnutsNoneNone
389A5E617A2-9831-3E9F-7072-6C5FB8129B(A0)129BHum-AlmBr024KM2(A0)Almond BranchesKnife Mill (2mm)RT vacuum sealedCmp17xk3Cmp17xk(A0)334.990002...2025-03-04 09:56:00NaTpassChemical compositionxkang2@lbl.govnot ready34None5Sweet potatoesNoneNone
\n", - "

390 rows × 21 columns

\n", "
" ], "text/plain": [ - " cmp_uuid_033 record_id prepared_sample \\\n", - "0 3EE2993D-86E3-1F16-C7EA-F8D555E114 (85)E114 Oak-TmPm01O(85) \n", - "1 46878EF9-1226-22A0-D5D8-CF65E241CB (85)41CB Oak-TmPm01O(85) \n", - "2 76A7A2F4-C4E4-E60F-1187-DEC6E02246 (85)2246 Oak-TmPm01O(85) \n", - "3 7A136832-286B-07CB-62DE-ACF52F9311 (85)9311 Oak-TmPm01O(85) \n", - "4 B709ECEE-F9A6-A55D-A59E-93B7B863D7 (85)63D7 Oak-TmPm01O(85) \n", - ".. ... ... ... \n", - "385 79799601-0E25-3832-3BBF-F4BD9EE973 (A0)E973 Hum-AlmBr024KM2(A0) \n", - "386 A4E3393F-72C3-A0ED-C041-105A33031A (A0)031A Hum-AlmBr024KM2(A0) \n", - "387 3D5A85F8-31A2-0CC2-74E6-05D8197C3A (A0)7C3A Hum-AlmBr024KM2(A0) \n", - "388 C7AE456E-4DCD-AE34-A0D0-BD39D04E42 (A0)4E42 Hum-AlmBr024KM2(A0) \n", - "389 A5E617A2-9831-3E9F-7072-6C5FB8129B (A0)129B Hum-AlmBr024KM2(A0) \n", - "\n", - " resource preparation_method storage_cond exper_abbrev \\\n", - "0 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "1 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "2 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "3 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - "4 Tomato pomace Oven dry RT vacuum sealed Cmp04xk \n", - ".. ... ... ... ... \n", - "385 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "386 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "387 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "388 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "389 Almond Branches Knife Mill (2mm) RT vacuum sealed Cmp17xk \n", - "\n", - " repl_no repl_id value ... created_at updated_at \\\n", - "0 1 Cmp04xk(85)1 14.16 ... 2025-01-23 09:00:01 NaT \n", - "1 2 Cmp04xk(85)2 14.18 ... 2025-01-23 09:00:16 NaT \n", - "2 3 Cmp04xk(85)3 14.12 ... 2025-01-23 09:00:31 NaT \n", - "3 1 Cmp04xk(85)1 15.74 ... 2025-01-23 09:00:46 NaT \n", - "4 2 Cmp04xk(85)2 15.75 ... 2025-01-23 09:01:01 NaT \n", - ".. ... ... ... ... ... ... \n", - "385 2 Cmp17xk(A0)2 17.290001 ... 2025-03-04 09:55:00 NaT \n", - "386 3 Cmp17xk(A0)3 17.5 ... 2025-03-04 09:55:15 NaT \n", - "387 1 Cmp17xk(A0)1 35.970001 ... 2025-03-04 09:55:30 NaT \n", - "388 2 Cmp17xk(A0)2 35.279999 ... 2025-03-04 09:55:45 NaT \n", - "389 3 Cmp17xk(A0)3 34.990002 ... 2025-03-04 09:56:00 NaT \n", - "\n", - " qc_result note analysis_type equipment raw_data_url \\\n", - "0 pass Chemical composition \n", - "1 pass Chemical composition \n", - "2 pass Chemical composition \n", - "3 pass Chemical composition \n", - "4 pass Chemical composition \n", - ".. ... ... ... ... ... \n", - "385 pass Chemical composition \n", - "386 pass Chemical composition \n", - "387 pass Chemical composition \n", - "388 pass Chemical composition \n", - "389 pass Chemical composition \n", - "\n", - " analyst_email upload_status parameter_id \n", - "0 xkang2@lbl.gov ready 6 \n", - "1 xkang2@lbl.gov ready 6 \n", - "2 xkang2@lbl.gov ready 6 \n", - "3 xkang2@lbl.gov ready 4 \n", - "4 xkang2@lbl.gov ready 4 \n", - ".. ... ... ... \n", - "385 xkang2@lbl.gov not ready 5 \n", - "386 xkang2@lbl.gov not ready 5 \n", - "387 xkang2@lbl.gov not ready 3 \n", - "388 xkang2@lbl.gov not ready 3 \n", - "389 xkang2@lbl.gov not ready 3 \n", - "\n", - "[390 rows x 21 columns]" + " description id name note uri\n", + "0 None 1 Tomatoes for processing None None\n", + "1 None 2 Grapes None None\n", + "2 None 3 Almonds None None\n", + "3 None 4 Walnuts None None\n", + "4 None 5 Sweet potatoes None None" ] }, - "execution_count": 15, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "# --- project root discovery (unchanged) ---\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# --- imports ---\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", + "\n", + "# --- query + dataframe ---\n", + "with Session(engine) as db:\n", + " stmt = select(*PrimaryAgProduct.__table__.columns)\n", + " rows = db.execute(stmt).mappings().all()\n", + "\n", + "df = pd.DataFrame(rows)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats = clean_dataframes[0].\\\n", + " groupby(['resource', 'parameter'])['value'].\\\n", + " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", + "\n", + "summary_stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", + " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", + " agg({'value': 'mean'}).\\\n", + " query('value > 30').\\\n", + " sort_values(by='value', ascending=False).\\\n", + " round({'value': 1})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", + "\n", + "def is_it_volatile_solids(df):\n", + " df['check'] = \"VS\"\n", + "\n", + " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", + " return df\n", + "\n", + "is_it_volatile_solids(df)\n", + "\n", + "df[['check', 'parameter']]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", + "def sqrtvalue(df):\n", + " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", + " return df\n", + "\n", + "#List comprehension to apply sqrtvalue to each DataFrame\n", + "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", + "\n", + "# Display the head of the third DataFrame\n", + "clean_rooted_df[2].head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cmpana_raw = cmpana.extract(project_root=project_root)\n", + "\n", + "cmpana_raw.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.orm import Session\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", + " replace_name_with_id_df,\n", + ") \n", + "\n", + "#This extractst the raw proximate data\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", + "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", + "\n", + "#this replaces the names with IDs\n", + "with Session(engine) as db:\n", + " parameter_ids = replace_name_with_id_df(\n", + " db=db,\n", + " df=test_df,\n", + " ref_model=Parameter,\n", + " name_column_name=\"name\", # column in df + table\n", + " id_column_name=\"id\", # PK column in table\n", + " final_column_name=\"parameter_id\"\n", + " )\n", + "\n", + "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", + "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", + "\n", + "#resource_id_mapping\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "parameter_ids\n" ] @@ -4187,6 +1461,63 @@ "\n", "field_sample" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] } ], "metadata": { diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py index d9290968..5d1d22b7 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/name_id_swap.py @@ -40,10 +40,13 @@ def replace_name_with_id_df( model_name_attr: str, id_column_name: str, final_column_name: str, -) -> pd.DataFrame: +) -> tuple[pd.DataFrame, int]: """ Replace a DataFrame name column with foreign key IDs from a SQLAlchemy table. Creates missing reference records if needed. + + Returns: + A tuple containing the modified DataFrame and the number of new records created. """ # 1. Fetch existing reference rows (name + id only) @@ -61,6 +64,7 @@ def replace_name_with_id_df( # 2. Determine which names are new unique_names = set(df_copy[df_name_column].dropna().unique()) new_names = unique_names - set(name_to_id_map.keys()) + num_new_records = len(new_names) # 3. Insert missing reference rows if new_names: @@ -87,4 +91,4 @@ def replace_name_with_id_df( df_copy[final_column_name] = df_copy[df_name_column].map(name_to_id_map) df_copy = df_copy.drop(columns=[df_name_column]) - return df_copy + return df_copy, num_new_records From f4b06ecae3b7e5f59fdb81f9bd288db4ede249f8 Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Tue, 6 Jan 2026 13:54:03 -0700 Subject: [PATCH 07/10] modified etl_notebook. Db is now running on localhost --- .../pipeline/utils/database_interaction.ipynb | 2 +- .../pipeline/utils/dev_test.ipynb | 53 ++ .../pipeline/utils/etl_notebook.ipynb | 462 +++++++++--------- 3 files changed, 291 insertions(+), 226 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb index 671f288a..c6ba4c99 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/database_interaction.ipynb @@ -347,7 +347,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.9" + "version": "3.13.11" } }, "nbformat": 4, diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb new file mode 100644 index 00000000..c7dfc663 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/dev_test.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d5df752a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ca_biositing.datamodels.schemas.generated.ca_biositing.Resource" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "import os\n", + "import sys\n", + "from sqlalchemy.orm import Session\n", + "\n", + "from ca_biositing.pipeline.utils.lookup_utils import replace_id_with_name_df\n", + "from ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb index 18fb0314..140d8709 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -20,15 +20,15 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-04 08:53:57,422 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", - "2026-01-04 08:53:57,423 - INFO - Successfully imported all project modules.\n" + "2026-01-06 10:31:31,052 - INFO - Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", + "2026-01-06 10:31:32,479 - INFO - Successfully imported all project modules.\n" ] } ], @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -128,12 +128,14 @@ " if 'updated_at' in df_cleaned.columns:\n", " df_cleaned['updated_at'] = pd.to_datetime(df_cleaned['updated_at'], errors='coerce')\n", "\n", - " # 4. Convert other dtypes to best possible\n", + " # 4. Replace empty strings with NaN so they are properly ignored\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "\n", + " # 5. Convert other dtypes to best possible\n", " df_cleaned = df_cleaned.convert_dtypes()\n", " logger.info('Successfully cleaned DataFrame.')\n", - " \n", - " \n", - " # 5. Convert all string data to lowercase\n", + "\n", + " # 6. Convert all string data to lowercase\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", " logger.info('Converted all string data to lowercase.')\n", " return df_cleaned\n", @@ -154,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -192,7 +194,7 @@ " \n", " try:\n", " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", - " df_normalized = replace_name_with_id_df(\n", + " df_normalized, num_created = replace_name_with_id_df(\n", " db=db,\n", " df=df_normalized,\n", " ref_model=model,\n", @@ -201,6 +203,8 @@ " id_column_name='id',\n", " final_column_name=f'{df_col}_id'\n", " )\n", + " if num_created > 0:\n", + " logger.info(f\"Created {num_created} new records in '{model.__name__}' table.\")\n", " new_col_name = f'{df_col}_id'\n", " num_nulls = df_normalized[new_col_name].isnull().sum()\n", " logger.info(f\"Successfully normalized '{df_col}'. New column '{new_col_name}' contains {num_nulls} null values.\")\n", @@ -219,7 +223,7 @@ " db.rollback()\n", " logger.info('Database session rolled back.')\n", " \n", - " return normalized_dfs" + " return normalized_dfs\n" ] }, { @@ -231,104 +235,112 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-04 08:58:52,849 - INFO - Starting data extraction...\n", - "2026-01-04 08:58:52,865 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:52,877 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:54,699 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:54,703 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:54,724 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:54,736 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:55,992 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:55,996 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:56,016 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-04 08:58:56,029 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-04 08:58:57,291 - INFO - Successfully extracted raw data.\n", - "2026-01-04 08:58:57,294 - INFO - Finished in state Completed()\n", - "2026-01-04 08:58:57,296 - INFO - Data extraction complete.\n", - "2026-01-04 08:58:57,299 - INFO - Starting data cleaning...\n", - "2026-01-04 08:58:57,299 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,302 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,309 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:40,834 - INFO - Starting data extraction...\n", + "2026-01-06 10:31:40,881 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:40,903 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:42,948 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/csrf-token?client=ae9812ec-079b-4fe5-845c-493121391421 \"HTTP/1.1 422 Unprocessable Entity\"\n", + "2026-01-06 10:31:42,966 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:45,659 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:45,661 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:45,672 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:45,680 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:46,995 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:47,292 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:47,296 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:47,313 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:31:47,326 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:31:48,924 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:31:48,929 - INFO - Finished in state Completed()\n", + "2026-01-06 10:31:48,930 - INFO - Data extraction complete.\n", + "2026-01-06 10:31:48,931 - INFO - Starting data cleaning...\n", + "2026-01-06 10:31:48,933 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,939 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,950 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,314 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,315 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,316 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,319 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:48,958 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,958 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,959 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,962 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,320 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,321 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-04 08:58:57,325 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-04 08:58:57,335 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_25314/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:31:48,964 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,964 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:31:48,965 - INFO - Dropped 0 rows with missing values.\n", + "2026-01-06 10:31:48,969 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-04 08:58:57,340 - INFO - Converted all string data to lowercase.\n", - "2026-01-04 08:58:57,342 - INFO - Data cleaning complete.\n", - "2026-01-04 08:58:57,342 - INFO - Starting data normalization...\n", - "2026-01-04 08:58:57,343 - INFO - Starting normalization process for 3 dataframes.\n", - "2026-01-04 08:58:57,343 - INFO - Processing DataFrame #1 with 759 rows.\n", - "2026-01-04 08:58:57,344 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,355 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,356 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,367 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,367 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,369 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,369 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,372 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,372 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,374 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,374 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,375 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,375 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,379 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,379 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,379 - INFO - Finished processing DataFrame #1.\n", - "2026-01-04 08:58:57,379 - INFO - Processing DataFrame #2 with 64 rows.\n", - "2026-01-04 08:58:57,380 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,384 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,384 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,388 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,388 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,392 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,392 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,396 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,396 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,398 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,398 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,399 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,400 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,403 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,403 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,403 - INFO - Finished processing DataFrame #2.\n", - "2026-01-04 08:58:57,403 - INFO - Processing DataFrame #3 with 390 rows.\n", - "2026-01-04 08:58:57,404 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-04 08:58:57,406 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-04 08:58:57,406 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-04 08:58:57,408 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-04 08:58:57,408 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-04 08:58:57,409 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-04 08:58:57,410 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-04 08:58:57,411 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-04 08:58:57,411 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-04 08:58:57,413 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-04 08:58:57,413 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-04 08:58:57,415 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-04 08:58:57,415 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-04 08:58:57,416 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-04 08:58:57,416 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", - "2026-01-04 08:58:57,417 - INFO - Finished processing DataFrame #3.\n", - "2026-01-04 08:58:57,417 - INFO - Committing database session.\n", - "2026-01-04 08:58:57,418 - INFO - Database commit successful.\n", - "2026-01-04 08:58:57,418 - INFO - Data normalization complete.\n", - "2026-01-04 08:58:57,418 - INFO - Displaying results of normalization...\n" + "2026-01-06 10:31:48,972 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:31:48,973 - INFO - Data cleaning complete.\n", + "2026-01-06 10:31:48,973 - INFO - Starting data normalization...\n", + "2026-01-06 10:31:48,974 - INFO - Starting normalization process for 3 dataframes.\n", + "2026-01-06 10:31:48,974 - INFO - Processing DataFrame #1 with 1030 rows.\n", + "2026-01-06 10:31:48,974 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,016 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", + "2026-01-06 10:31:49,116 - INFO - Created 35 new records in 'Resource' table.\n", + "2026-01-06 10:31:49,116 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,117 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,120 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,120 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,123 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,124 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,144 - INFO - Created 1 new records in 'Parameter' table.\n", + "2026-01-06 10:31:49,145 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,147 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,167 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,176 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,182 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,182 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,189 - INFO - Created 1 new records in 'AnalysisType' table.\n", + "2026-01-06 10:31:49,189 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,189 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,190 - INFO - Finished processing DataFrame #1.\n", + "2026-01-06 10:31:49,190 - INFO - Processing DataFrame #2 with 64 rows.\n", + "2026-01-06 10:31:49,191 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,197 - INFO - Created 1 new records in 'Resource' table.\n", + "2026-01-06 10:31:49,198 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,198 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,201 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,202 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,204 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,204 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,206 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,206 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,209 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,209 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,211 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,212 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,213 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,213 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,214 - INFO - Finished processing DataFrame #2.\n", + "2026-01-06 10:31:49,214 - INFO - Processing DataFrame #3 with 390 rows.\n", + "2026-01-06 10:31:49,215 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:31:49,217 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:31:49,217 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:31:49,220 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:31:49,220 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:31:49,222 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:31:49,222 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:31:49,224 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:31:49,224 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:31:49,225 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:31:49,225 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:31:49,228 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:31:49,228 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:31:49,230 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:31:49,230 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", + "2026-01-06 10:31:49,230 - INFO - Finished processing DataFrame #3.\n", + "2026-01-06 10:31:49,230 - INFO - Committing database session.\n", + "2026-01-06 10:31:49,232 - INFO - Database commit successful.\n", + "2026-01-06 10:31:49,241 - INFO - Data normalization complete.\n", + "2026-01-06 10:31:49,244 - INFO - Displaying results of normalization...\n" ] }, { @@ -448,7 +460,7 @@ "type": "integer" } ], - "ref": "e2d8eb1c-3fc8-4b97-99b0-df76b598d9df", + "ref": "b0e10f3c-2f75-4628-acc4-1c383e10ca93", "rows": [ [ "0", @@ -465,13 +477,13 @@ "pass", "not ready", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "1", @@ -488,13 +500,13 @@ "pass", "ready", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "2", @@ -511,13 +523,13 @@ "pass", "imported", "", - "24", + "6", "109", "7", - "25", + "3", "1", "1", - "6" + "1" ], [ "3", @@ -534,13 +546,13 @@ "pass", "import failed", "", - "24", + "6", "109", "7", - "23", + "4", "1", "1", - "6" + "1" ], [ "4", @@ -557,13 +569,13 @@ "pass", "", "", - "24", + "6", "109", "7", - "23", + "4", "1", "1", - "6" + "1" ] ], "shape": { @@ -628,13 +640,13 @@ " pass\n", " not ready\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 1\n", @@ -651,13 +663,13 @@ " pass\n", " ready\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 2\n", @@ -674,13 +686,13 @@ " pass\n", " imported\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 25\n", + " 3\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 3\n", @@ -697,13 +709,13 @@ " pass\n", " import failed\n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 23\n", + " 4\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", " 4\n", @@ -720,13 +732,13 @@ " pass\n", " \n", " \n", - " 24\n", + " 6\n", " 109\n", " 7\n", - " 23\n", + " 4\n", + " 1\n", " 1\n", " 1\n", - " 6\n", " \n", " \n", "\n", @@ -748,25 +760,25 @@ "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", "\n", " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", - "0 NaT pass not ready 24 109 \n", - "1 NaT pass ready 24 109 \n", - "2 NaT pass imported 24 109 \n", - "3 NaT pass import failed 24 109 \n", - "4 NaT pass 24 109 \n", + "0 NaT pass not ready 6 109 \n", + "1 NaT pass ready 6 109 \n", + "2 NaT pass imported 6 109 \n", + "3 NaT pass import failed 6 109 \n", + "4 NaT pass 6 109 \n", "\n", " preparation_method_id parameter_id unit_id analyst_email_id \\\n", - "0 7 25 1 1 \n", - "1 7 25 1 1 \n", - "2 7 25 1 1 \n", - "3 7 23 1 1 \n", - "4 7 23 1 1 \n", + "0 7 3 1 1 \n", + "1 7 3 1 1 \n", + "2 7 3 1 1 \n", + "3 7 4 1 1 \n", + "4 7 4 1 1 \n", "\n", " analysis_type_id \n", - "0 6 \n", - "1 6 \n", - "2 6 \n", - "3 6 \n", - "4 6 " + "0 1 \n", + "1 1 \n", + "2 1 \n", + "3 1 \n", + "4 1 " ] }, "metadata": {}, @@ -899,7 +911,7 @@ "type": "integer" } ], - "ref": "323960ff-b723-432d-b7c9-fa0f9dd8c185", + "ref": "0f732e49-22cb-4b72-a52a-9fade9af05ef", "rows": [ [ "0", @@ -918,13 +930,13 @@ "", "", "", - "8", + "27", "80", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "1", @@ -943,13 +955,13 @@ "", "", "", - "8", + "27", "80", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "2", @@ -968,13 +980,13 @@ "", "", "", - "26", + "11", "96", "6", - "28", + "8", "3", "2", - "7" + "2" ], [ "3", @@ -993,13 +1005,13 @@ "", "", "", - "2", + "12", "91", "9", - "28", + "8", "3", "2", - "7" + "2" ], [ "4", @@ -1018,13 +1030,13 @@ "", "", "", - "7", + "21", "153", "9", - "28", + "8", "3", "2", - "7" + "2" ] ], "shape": { @@ -1091,13 +1103,13 @@ " \n", " \n", " \n", - " 8\n", + " 27\n", " 80\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 1\n", @@ -1115,13 +1127,13 @@ " \n", " \n", " \n", - " 8\n", + " 27\n", " 80\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 2\n", @@ -1139,13 +1151,13 @@ " \n", " \n", " \n", - " 26\n", + " 11\n", " 96\n", " 6\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 3\n", @@ -1163,13 +1175,13 @@ " \n", " \n", " \n", - " 2\n", + " 12\n", " 91\n", " 9\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", " 4\n", @@ -1187,13 +1199,13 @@ " \n", " \n", " \n", - " 7\n", + " 21\n", " 153\n", " 9\n", - " 28\n", + " 8\n", " 3\n", " 2\n", - " 7\n", + " 2\n", " \n", " \n", "\n", @@ -1216,25 +1228,25 @@ "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", "\n", " updated_at ... equipment raw_data_url upload_status resource_id \\\n", - "0 NaT ... 8 \n", - "1 NaT ... 8 \n", - "2 NaT ... 26 \n", - "3 NaT ... 2 \n", - "4 NaT ... 7 \n", + "0 NaT ... 27 \n", + "1 NaT ... 27 \n", + "2 NaT ... 11 \n", + "3 NaT ... 12 \n", + "4 NaT ... 21 \n", "\n", " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 80 6 28 3 \n", - "1 80 6 28 3 \n", - "2 96 6 28 3 \n", - "3 91 9 28 3 \n", - "4 153 9 28 3 \n", + "0 80 6 8 3 \n", + "1 80 6 8 3 \n", + "2 96 6 8 3 \n", + "3 91 9 8 3 \n", + "4 153 9 8 3 \n", "\n", " analyst_email_id analysis_type_id \n", - "0 2 7 \n", - "1 2 7 \n", - "2 2 7 \n", - "3 2 7 \n", - "4 2 7 \n", + "0 2 2 \n", + "1 2 2 \n", + "2 2 2 \n", + "3 2 2 \n", + "4 2 2 \n", "\n", "[5 rows x 22 columns]" ] @@ -1364,7 +1376,7 @@ "type": "integer" } ], - "ref": "31e4fb08-8acb-41fc-bdec-4f79aea24d5f", + "ref": "95361c64-4a65-4fd6-bde6-2efae692d84c", "rows": [ [ "0", @@ -1382,10 +1394,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1406,10 +1418,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1430,10 +1442,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "12", + "13", "2", "1", "3" @@ -1454,10 +1466,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "14", + "11", "2", "1", "3" @@ -1478,10 +1490,10 @@ "", "", "ready", - "24", + "6", "92", "8", - "14", + "11", "2", "1", "3" @@ -1551,10 +1563,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1575,10 +1587,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1599,10 +1611,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 12\n", + " 13\n", " 2\n", " 1\n", " 3\n", @@ -1623,10 +1635,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 14\n", + " 11\n", " 2\n", " 1\n", " 3\n", @@ -1647,10 +1659,10 @@ " \n", " \n", " ready\n", - " 24\n", + " 6\n", " 92\n", " 8\n", - " 14\n", + " 11\n", " 2\n", " 1\n", " 3\n", @@ -1676,18 +1688,18 @@ "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", "\n", " qc_result ... equipment raw_data_url upload_status resource_id \\\n", - "0 pass ... ready 24 \n", - "1 pass ... ready 24 \n", - "2 pass ... ready 24 \n", - "3 pass ... ready 24 \n", - "4 pass ... ready 24 \n", + "0 pass ... ready 6 \n", + "1 pass ... ready 6 \n", + "2 pass ... ready 6 \n", + "3 pass ... ready 6 \n", + "4 pass ... ready 6 \n", "\n", " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 92 8 12 2 \n", - "1 92 8 12 2 \n", - "2 92 8 12 2 \n", - "3 92 8 14 2 \n", - "4 92 8 14 2 \n", + "0 92 8 13 2 \n", + "1 92 8 13 2 \n", + "2 92 8 13 2 \n", + "3 92 8 11 2 \n", + "4 92 8 11 2 \n", "\n", " analyst_email_id analysis_type_id \n", "0 1 3 \n", @@ -1765,9 +1777,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Pixi Default Environment", + "display_name": "default", "language": "python", - "name": "pixi-default" + "name": "python3" }, "language_info": { "codemirror_mode": { From 5f1e0a410a0fd5b5b4f89e21beb96ea1d10738ba Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Wed, 7 Jan 2026 09:32:28 -0700 Subject: [PATCH 08/10] "did the module refactor but may need to mess with the alembic env.py. Committing before that" --- .../datamodels/linkml/ca_biositing.yaml | 112 +- .../linkml/modules/aim1_records.yaml | 140 -- .../aim1_records/aim1_record_base.yaml | 86 + .../aim1_records/calorimetry_record.yaml | 21 + .../aim1_records/compositional_record.yaml | 21 + .../modules/aim1_records/ftnir_record.yaml | 21 + .../icp_record.yaml} | 16 +- .../aim1_records/proximate_record.yaml | 21 + .../modules/aim1_records/rgb_record.yaml | 21 + .../modules/aim1_records/ultimate_record.yaml | 21 + .../modules/aim1_records/xrd_record.yaml | 32 + .../modules/aim1_records/xrf_record.yaml | 40 + .../linkml/modules/aim2_records.yaml | 170 -- .../aim2_records/aim2_record_base.yaml | 60 + .../aim2_records/autoclave_record.yaml | 22 + .../aim2_records/fermentation_record.yaml | 58 + .../aim2_records/gasifcation_record.yaml | 0 .../aim2_records/gasification_record.yaml | 36 + .../aim2_records/pretreatment_record.yaml | 49 + .../linkml/modules/aim2_records/strain.yaml | 27 + .../datamodels/linkml/modules/core.yaml | 79 - .../linkml/modules/core/base_entity.yaml | 40 + .../linkml/modules/core/lookup_base.yaml | 34 + .../datamodels/linkml/modules/core/note.yaml | 12 + .../data_source.yaml} | 83 +- .../data_source_type.yaml | 21 + .../file_object_metadata.yaml | 46 + .../location_resolution.yaml | 13 + .../data_sources_metadata/source_type.yaml | 13 + .../linkml/modules/experiment_equipment.yaml | 127 -- .../experiment_equipment/equipment.yaml | 21 + .../experiment_equipment/experiment.yaml | 44 + .../experiment_analysis.yaml | 30 + .../experiment_equipment.yaml | 30 + .../experiment_method.yaml | 31 + .../experiment_prepared_sample.yaml | 31 + .../linkml/modules/external_data.yaml | 441 ---- .../billion_ton_2023_record.yaml | 89 + .../modules/external_data/landiq_record.yaml | 69 + .../linkml/modules/external_data/polygon.yaml | 30 + .../resource_usda_commodity_map.yaml | 39 + .../external_data/usda_census_record.yaml | 44 + .../modules/external_data/usda_commodity.yaml | 30 + .../modules/external_data/usda_domain.yaml | 13 + .../external_data/usda_market_record.yaml | 119 ++ .../external_data/usda_market_report.yaml | 57 + .../usda_statistic_category.yaml | 13 + .../external_data/usda_survey_program.yaml | 13 + .../external_data/usda_survey_record.yaml | 60 + .../modules/external_data/usda_term_map.yaml | 42 + .../linkml/modules/field_sampling.yaml | 200 -- .../modules/field_sampling/ag_treatment.yaml | 13 + .../field_sampling/collection_method.yaml | 13 + .../modules/field_sampling/field_sample.yaml | 105 + .../field_sample_condition.yaml | 43 + .../field_sampling/field_storage_method.yaml | 13 + .../field_sampling/harvest_method.yaml | 13 + .../field_sampling/location_soil_type.yaml | 27 + .../physical_characteristic.yaml | 42 + .../field_sampling/processing_method.yaml | 13 + .../modules/field_sampling/soil_type.yaml | 13 + .../linkml/modules/general_analysis.yaml | 97 - .../general_analysis/analysis_type.yaml | 13 + .../modules/general_analysis/dataset.yaml | 38 + .../general_analysis/dimension_type.yaml | 13 + .../modules/general_analysis/observation.yaml | 64 + .../datamodels/linkml/modules/geography.yaml | 109 - .../linkml/modules/infrastructure.yaml | 1069 ---------- .../infrastructure/facility_record.yaml | 54 + .../infrastructure_biodiesel_plants.yaml | 83 + .../infrastructure_biosolids_facilities.yaml | 228 +++ .../infrastructure_cafo_manure_locations.yaml | 98 + .../infrastructure_combustion_plants.yaml | 103 + ...nfrastructure_district_energy_systems.yaml | 123 ++ .../infrastructure_ethanol_biorefineries.yaml | 63 + ...astructure_food_processing_facilities.yaml | 103 + .../infrastructure_landfills.yaml | 188 ++ ...ructure_livestock_anaerobic_digesters.yaml | 133 ++ ...ure_msw_to_energy_anaerobic_digesters.yaml | 98 + ...cture_saf_and_renewable_diesel_plants.yaml | 83 + ...structure_wastewater_treatment_plants.yaml | 168 ++ .../datamodels/linkml/modules/lineage.yaml | 88 - .../modules/lineage/entity_lineage.yaml | 36 + .../linkml/modules/lineage/etl_run.yaml | 44 + .../linkml/modules/lineage/lineage_group.yaml | 25 + .../modules/methods_parameters_units.yaml | 130 -- .../methods_parameters_units/method.yaml | 46 + .../method_abbrev.yaml | 13 + .../method_category.yaml | 13 + .../method_standard.yaml | 13 + .../methods_parameters_units/parameter.yaml | 28 + .../parameter_category.yaml | 13 + .../parameter_category_parameter.yaml | 31 + .../parameter_unit.yaml | 31 + .../methods_parameters_units/unit.yaml | 13 + .../{people.yaml => people/contact.yaml} | 30 +- .../linkml/modules/people/provider.yaml | 24 + .../datamodels/linkml/modules/places.yaml | 105 - .../linkml/modules/places/geography.yaml | 52 + .../modules/places/location_address.yaml | 56 + .../linkml/modules/primary_ag_product.yaml | 23 - .../linkml/modules/resource_availability.yaml | 35 - .../modules/resource_counterfactual.yaml | 47 - .../linkml/modules/resource_information.yaml | 189 -- .../primary_ag_product.yaml | 14 + .../resource_information/resource.yaml | 44 + .../resource_availability.yaml | 43 + .../resource_information/resource_class.yaml | 13 + .../resource_counterfactual.yaml | 96 + .../resource_morphology.yaml | 19 +- .../resource_subclass.yaml | 13 + .../linkml/modules/resource_subclass.yaml | 21 - .../linkml/modules/sample_preparation.yaml | 70 - .../preparation_method.yaml | 34 + .../preparation_method_abbreviation.yaml | 13 + .../sample_preparation/prepared_sample.yaml | 40 + .../linkml/test_schemas/schema1.yaml | 18 + .../linkml/test_schemas/schema2.yaml | 22 + .../linkml/test_schemas/schema3.yaml | 26 + .../schemas/generated/aim1_records.py | 1747 ---------------- .../schemas/generated/aim2_records.py | 1600 --------------- .../schemas/generated/ca_biositing.py | 1817 ++++++++--------- .../datamodels/schemas/generated/core.py | 46 - .../generated/data_sources_metadata.py | 1030 ---------- .../schemas/generated/experiment_equipment.py | 1371 ------------- .../schemas/generated/external_data.py | 1557 -------------- .../schemas/generated/field_sampling.py | 1030 ---------- .../schemas/generated/general_analysis.py | 1148 ----------- .../datamodels/schemas/generated/geography.py | 106 - .../schemas/generated/infrastructure.py | 1625 --------------- .../datamodels/schemas/generated/lineage.py | 109 - .../generated/methods_parameters_units.py | 1030 ---------- .../datamodels/schemas/generated/people.py | 106 - .../datamodels/schemas/generated/places.py | 106 - .../schemas/generated/primary_ag_product.py | 75 - .../generated/resource_availability.py | 308 --- .../schemas/generated/resource_class.py | 74 - .../generated/resource_counterfactual.py | 1030 ---------- .../schemas/generated/resource_information.py | 308 --- .../schemas/generated/resource_morphology.py | 308 --- .../schemas/generated/resource_subclass.py | 74 - .../schemas/generated/sample_preparation.py | 1119 ---------- .../schemas/generated_test/schema1.py | 22 + .../schemas/generated_test/schema2.py | 40 + .../schemas/generated_test/schema3.py | 59 + .../datamodels/utils/generate_sqla.py | 25 +- .../datamodels/utils/generate_test_sqla.py | 69 + .../pipeline/utils/etl_notebook.ipynb | 672 +++--- 148 files changed, 5824 insertions(+), 20469 deletions(-) delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/aim1_record_base.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/calorimetry_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/compositional_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ftnir_record.yaml rename src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/{resource_class.yaml => aim1_records/icp_record.yaml} (51%) create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/proximate_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/rgb_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ultimate_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrd_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrf_record.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/aim2_record_base.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/autoclave_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/fermentation_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasifcation_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasification_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/pretreatment_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/strain.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/base_entity.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/lookup_base.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/note.yaml rename src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/{data_sources_metadata.yaml => data_sources_metadata/data_source.yaml} (55%) create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source_type.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/file_object_metadata.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/location_resolution.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/source_type.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/equipment.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_analysis.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_equipment.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_prepared_sample.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/billion_ton_2023_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/landiq_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/polygon.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/resource_usda_commodity_map.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_census_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_commodity.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_domain.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_report.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_statistic_category.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_program.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_term_map.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/ag_treatment.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/collection_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample_condition.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_storage_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/harvest_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/location_soil_type.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/physical_characteristic.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/processing_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/soil_type.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/analysis_type.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dataset.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dimension_type.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/observation.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/geography.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/facility_record.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biodiesel_plants.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biosolids_facilities.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_cafo_manure_locations.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_combustion_plants.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_district_energy_systems.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_ethanol_biorefineries.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_food_processing_facilities.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_landfills.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_livestock_anaerobic_digesters.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_saf_and_renewable_diesel_plants.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_wastewater_treatment_plants.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/entity_lineage.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/etl_run.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/lineage_group.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_abbrev.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_category.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_standard.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category_parameter.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_unit.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/unit.yaml rename src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/{people.yaml => people/contact.yaml} (53%) create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/provider.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/geography.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/location_address.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/primary_ag_product.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_availability.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_class.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_counterfactual.yaml rename src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/{ => resource_information}/resource_morphology.yaml (58%) create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_subclass.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method_abbreviation.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/prepared_sample.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema1.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema2.yaml create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema3.yaml delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/core.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/geography.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/lineage.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/people.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/places.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py delete mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema1.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema2.py create mode 100644 src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema3.py create mode 100644 src/ca_biositing/datamodels/utils/generate_test_sqla.py diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/ca_biositing.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/ca_biositing.yaml index fed17afb..5c2c2bd9 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/ca_biositing.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/ca_biositing.yaml @@ -16,19 +16,99 @@ default_range: string imports: - linkml:types - - modules/core - - modules/resource_information - - modules/field_sampling - - modules/places - - modules/people - - modules/lineage - - modules/data_sources_metadata - - modules/sample_preparation - - modules/methods_parameters_units - - modules/experiment_equipment - - modules/general_analysis - - modules/aim1_records - - modules/aim2_records - - modules/external_data - - modules/infrastructure - - modules/geography + - modules/core/base_entity + - modules/core/lookup_base + - modules/core/note + - modules/data_sources_metadata/data_source + - modules/data_sources_metadata/data_source_type + - modules/data_sources_metadata/file_object_metadata + - modules/data_sources_metadata/location_resolution + - modules/data_sources_metadata/source_type + - modules/experiment_equipment/equipment + - modules/experiment_equipment/experiment + - modules/experiment_equipment/experiment_analysis + - modules/experiment_equipment/experiment_equipment + - modules/experiment_equipment/experiment_method + - modules/experiment_equipment/experiment_prepared_sample + - modules/external_data/billion_ton_2023_record + - modules/external_data/landiq_record + - modules/external_data/polygon + - modules/external_data/resource_usda_commodity_map + - modules/external_data/usda_census_record + - modules/external_data/usda_commodity + - modules/external_data/usda_domain + - modules/external_data/usda_market_record + - modules/external_data/usda_market_report + - modules/external_data/usda_statistic_category + - modules/external_data/usda_survey_program + - modules/external_data/usda_survey_record + - modules/external_data/usda_term_map + - modules/field_sampling/ag_treatment + - modules/field_sampling/collection_method + - modules/field_sampling/field_sample + - modules/field_sampling/field_sample_condition + - modules/field_sampling/field_storage_method + - modules/field_sampling/harvest_method + - modules/field_sampling/location_soil_type + - modules/field_sampling/physical_characteristic + - modules/field_sampling/processing_method + - modules/field_sampling/soil_type + - modules/general_analysis/analysis_type + - modules/general_analysis/dataset + - modules/general_analysis/dimension_type + - modules/general_analysis/observation + - modules/infrastructure/facility_record + - modules/infrastructure/infrastructure_biodiesel_plants + - modules/infrastructure/infrastructure_biosolids_facilities + - modules/infrastructure/infrastructure_cafo_manure_locations + - modules/infrastructure/infrastructure_combustion_plants + - modules/infrastructure/infrastructure_district_energy_systems + - modules/infrastructure/infrastructure_ethanol_biorefineries + - modules/infrastructure/infrastructure_food_processing_facilities + - modules/infrastructure/infrastructure_landfills + - modules/infrastructure/infrastructure_livestock_anaerobic_digesters + - modules/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters + - modules/infrastructure/infrastructure_saf_and_renewable_diesel_plants + - modules/infrastructure/infrastructure_wastewater_treatment_plants + - modules/lineage/entity_lineage + - modules/lineage/etl_run + - modules/lineage/lineage_group + - modules/methods_parameters_units/method + - modules/methods_parameters_units/method_abbrev + - modules/methods_parameters_units/method_category + - modules/methods_parameters_units/method_standard + - modules/methods_parameters_units/parameter + - modules/methods_parameters_units/parameter_category + - modules/methods_parameters_units/parameter_category_parameter + - modules/methods_parameters_units/parameter_unit + - modules/methods_parameters_units/unit + - modules/people/contact + - modules/people/provider + - modules/places/geography + - modules/places/location_address + - modules/aim1_records/aim1_record_base + - modules/aim1_records/proximate_record + - modules/aim1_records/ultimate_record + - modules/aim1_records/compositional_record + - modules/aim1_records/icp_record + - modules/aim1_records/xrf_record + - modules/aim1_records/xrd_record + - modules/aim1_records/calorimetry_record + - modules/aim1_records/ftnir_record + - modules/aim1_records/rgb_record + - modules/aim2_records/aim2_record_base + - modules/aim2_records/pretreatment_record + - modules/aim2_records/fermentation_record + - modules/aim2_records/gasification_record + - modules/aim2_records/autoclave_record + - modules/aim2_records/strain + - modules/resource_information/primary_ag_product + - modules/resource_information/resource + - modules/resource_information/resource_availability + - modules/resource_information/resource_class + #- modules/resource_information/resource_counterfactual + - modules/resource_information/resource_morphology + - modules/resource_information/resource_subclass + - modules/sample_preparation/preparation_method + - modules/sample_preparation/preparation_method_abbreviation + - modules/sample_preparation/prepared_sample diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records.yaml deleted file mode 100644 index 8328faec..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records.yaml +++ /dev/null @@ -1,140 +0,0 @@ -id: https://w3id.org/ca_biositing/aim1_records -name: aim1_records -title: Aim 1 Records -description: Records for Aim 1 analysis. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - general_analysis - - experiment_equipment - - resource_information - - sample_preparation - - methods_parameters_units - - data_sources_metadata - -classes: - Aim1RecordBase: - is_a: BaseEntity - abstract: true - slots: - - dataset_id - - experiment_id - - resource_id - - sample_id - - technical_replicate_no - - technical_replicate_total - - method_id - - analyst_id - - raw_data_id - - qc_pass - - note - - ProximateRecord: - is_a: Aim1RecordBase - description: Proximate analysis record. - - UltimateRecord: - is_a: Aim1RecordBase - description: Ultimate analysis record. - - CompositionalRecord: - is_a: Aim1RecordBase - description: Compositional analysis record. - - IcpRecord: - is_a: Aim1RecordBase - description: ICP analysis record. - - XrfRecord: - is_a: Aim1RecordBase - description: XRF analysis record. - slots: - - maybe_wavelength_nm - - maybe_intensity - - maybe_energy_slope - - maybe_energy_offset - - XrdRecord: - is_a: Aim1RecordBase - description: XRD analysis record. - slots: - - maybe_scan_low_nm - - maybe_scan_high_nm - - CalorimetryRecord: - is_a: Aim1RecordBase - description: Calorimetry analysis record. - - FtnirRecord: - is_a: Aim1RecordBase - description: FT-NIR analysis record. - - RgbRecord: - is_a: Aim1RecordBase - description: RGB analysis record. - -slots: - #Slots imported - #dataset_id: from general_analysis.yaml - #experiment_id: from experiments_equipment.yaml - #analyst_id: from experiments_equipment.yaml - #resource_id: from resource_information.yaml - #method_id: from experiments_equipment.yaml - #note: from core.yaml - - #New slots defined in Aim1RecordBase class - - sample_id: - range: integer - description: Reference to PreparedSample. - - technical_replicate_no: - range: integer - description: Replicate number. - - technical_replicate_total: - range: integer - description: Total number of replicates. - - raw_data_id: - range: integer - description: Reference to FileObjectMetadata. - - qc_pass: - range: boolean - description: Whether the record passed QC. - - #Potential slots for XrfRecord and XrdRecords - - maybe_wavelength_nm: - range: decimal - description: Wavelength in nm. - - maybe_intensity: - range: decimal - description: Intensity. - - maybe_energy_slope: - range: decimal - description: Energy slope. - - maybe_energy_offset: - range: decimal - description: Energy offset. - - maybe_scan_low_nm: - range: integer - description: Low scan range in nm. - - maybe_scan_high_nm: - range: integer - description: High scan range in nm. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/aim1_record_base.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/aim1_record_base.yaml new file mode 100644 index 00000000..0a6d0b20 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/aim1_record_base.yaml @@ -0,0 +1,86 @@ +id: https://w3id.org/ca_biositing/aim1_records/aim1_record_base +name: aim1_record_base +title: Aim 1 Record Base +description: Base class for Aim 1 records. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - ../core/base_entity + - ../general_analysis/dataset + - ../experiment_equipment/experiment + - ../resource_information/resource + - ../sample_preparation/prepared_sample + - ../methods_parameters_units/method + - ../people/contact + - ../data_sources_metadata/file_object_metadata + +classes: + Aim1RecordBase: + is_a: BaseEntity + abstract: true + slots: + - dataset_id + + - experiment_id + - resource_id + - sample_id + - technical_replicate_no + - technical_replicate_total + - method_id + - analyst_id + - raw_data_id + - qc_pass + - note + +slots: + sample_id: + range: integer + description: Reference to PreparedSample. + + resource_id: + range: Resource + description: Reference to Resource. + + method_id: + range: Method + description: Reference to Method used in analysis. + + analyst_id: + range: Contact + description: Reference to Analyst who performed analysis. + + raw_data_id: + range: FileObjectMetadata + description: Reference to FileObjectMetadata. + + dataset_id: + range: Dataset + description: Reference to Dataset. + + experiment_id: + range: Experiment + description: Reference to Experiment. + + technical_replicate_no: + range: integer + description: Replicate number. + + technical_replicate_total: + range: integer + description: Total number of replicates. + + qc_pass: + range: boolean + description: Whether the record passed QC. + + note: + range: string + description: Additional notes about the record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/calorimetry_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/calorimetry_record.yaml new file mode 100644 index 00000000..57de7d67 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/calorimetry_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/calorimetry_record +name: calorimetry_record +title: Calorimetry Record +description: Calorimetry analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + CalorimetryRecord: + is_a: Aim1RecordBase + description: Calorimetry analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/compositional_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/compositional_record.yaml new file mode 100644 index 00000000..0deffcb2 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/compositional_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/compositional_record +name: compositional_record +title: Compositional Record +description: Compositional analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + CompositionalRecord: + is_a: Aim1RecordBase + description: Compositional analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ftnir_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ftnir_record.yaml new file mode 100644 index 00000000..e9386428 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ftnir_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/ftnir_record +name: ftnir_record +title: FT-NIR Record +description: FT-NIR analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + FtnirRecord: + is_a: Aim1RecordBase + description: FT-NIR analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/icp_record.yaml similarity index 51% rename from src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml rename to src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/icp_record.yaml index cc39fe4e..db991695 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_class.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/icp_record.yaml @@ -1,7 +1,7 @@ -id: https://w3id.org/ca_biositing/resource_class -name: resource_class -title: Resource Class -description: Classification of resources. +id: https://w3id.org/ca_biositing/aim1_records/icp_record +name: icp_record +title: ICP Record +description: ICP analysis record. license: https://creativecommons.org/publicdomain/zero/1.0/ prefixes: @@ -13,9 +13,9 @@ default_range: string imports: - linkml:types - - core + - aim1_record_base classes: - ResourceClass: - is_a: LookupBase - description: Classification of resources. + IcpRecord: + is_a: Aim1RecordBase + description: ICP analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/proximate_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/proximate_record.yaml new file mode 100644 index 00000000..57e04b15 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/proximate_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/proximate_record +name: proximate_record +title: Proximate Record +description: Proximate analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + ProximateRecord: + is_a: Aim1RecordBase + description: Proximate analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/rgb_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/rgb_record.yaml new file mode 100644 index 00000000..11dc81bc --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/rgb_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/rgb_record +name: rgb_record +title: RGB Record +description: RGB analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + RgbRecord: + is_a: Aim1RecordBase + description: RGB analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ultimate_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ultimate_record.yaml new file mode 100644 index 00000000..b03835de --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/ultimate_record.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/aim1_records/ultimate_record +name: ultimate_record +title: Ultimate Record +description: Ultimate analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + UltimateRecord: + is_a: Aim1RecordBase + description: Ultimate analysis record. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrd_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrd_record.yaml new file mode 100644 index 00000000..00a69f73 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrd_record.yaml @@ -0,0 +1,32 @@ +id: https://w3id.org/ca_biositing/aim1_records/xrd_record +name: xrd_record +title: XRD Record +description: XRD analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + XrdRecord: + is_a: Aim1RecordBase + description: XRD analysis record. + slots: + - scan_low_nm + - scan_high_nm + +slots: + scan_low_nm: + range: integer + description: Low scan range in nm. + scan_high_nm: + range: integer + description: High scan range in nm. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrf_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrf_record.yaml new file mode 100644 index 00000000..df026e1b --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim1_records/xrf_record.yaml @@ -0,0 +1,40 @@ +id: https://w3id.org/ca_biositing/aim1_records/xrf_record +name: xrf_record +title: XRF Record +description: XRF analysis record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim1_record_base + +classes: + XrfRecord: + is_a: Aim1RecordBase + description: XRF analysis record. + slots: + - wavelength_nm + - intensity + - energy_slope + - energy_offset + +slots: + wavelength_nm: + range: decimal + description: Wavelength in nm. + intensity: + range: decimal + description: Intensity. + energy_slope: + range: decimal + description: Energy slope. + energy_offset: + range: decimal + description: Energy offset. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records.yaml deleted file mode 100644 index 8e20f1c9..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records.yaml +++ /dev/null @@ -1,170 +0,0 @@ -id: https://w3id.org/ca_biositing/aim2_records -name: aim2_records -title: Aim 2 Records -description: Records for Aim 2 analysis. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - general_analysis - - experiment_equipment - - resource_information - - sample_preparation - - methods_parameters_units - - data_sources_metadata - - people - -classes: - Aim2RecordBase: - is_a: BaseEntity - abstract: true - slots: - - dataset_id - - experiment_id - - resource_id - - sample_id - - technical_replicate_total - - technical_replicate_no - - calc_method_id - - calc_analyst_id - - raw_data_id - - qc_pass - - note - - PretreatmentRecord: - is_a: Aim2RecordBase - description: Pretreatment record. - slots: - - pretreatment_method_id - - eh_method_id - - reaction_block_id - - block_position - - temperature - - replicate_no #delete bc redundancy with technical_replicate_no in aim2recordbase entity? - - FermentationRecord: - is_a: Aim2RecordBase - description: Fermentation record. - slots: - - strain_id - - pretreatment_method_id - - eh_method_id - - replicate_no #delete bc redundancy with technical_replicate_no in aim2recordbase entity? - - well_position - - temperature - - agitation_rpm - - vessel_id - - analyte_detection_equipment_id - - GasificationRecord: - is_a: Aim2RecordBase - description: Gasification record. - slots: - - feedstock_mass - - bed_temperature - - gas_flow_rate - - AutoclaveRecord: - is_a: Aim2RecordBase - description: Autoclave record. - slots: [] - - Strain: - is_a: LookupBase - description: Strain used in fermentation. - slots: - - parent_strain_id - -slots: - pretreatment_method_id: - range: integer - description: Reference to Method (pretreatment). - - eh_method_id: - range: integer - description: Reference to Method (enzymatic hydrolysis). - - reaction_block_id: - range: integer - description: ID of the reaction block. - - block_position: - range: string - description: Position in the block. - - temperature: - range: decimal - description: Temperature. - - replicate_no: - range: integer - description: Replicate number. - - strain_id: - range: integer - description: Reference to Strain. - - well_position: - range: string - description: Position in the well. - - agitation_rpm: - range: decimal - description: Agitation in RPM. - - vessel_id: - range: integer - description: Reference to Equipment (vessel). - - analyte_detection_equipment_id: - range: integer - description: Reference to Equipment (detection). - - feedstock_mass: - range: decimal - description: Mass of feedstock. - - bed_temperature: - range: decimal - description: Bed temperature. - - gas_flow_rate: - range: decimal - description: Gas flow rate. - - parent_strain_id: - range: integer - - sample_id: - range: integer - - raw_data_id: - range: integer - description: Reference to FileObjectMetadata. - - qc_pass: - range: boolean - description: Whether the record passed QC. - - technical_replicate_no: - range: integer - description: Replicate number. - - calc_method_id: - range: integer - description: Reference to Method used in calculation. - - calc_analyst_id: - range: integer - description: Reference to Analyst who performed calculation. - - technical_replicate_total: - range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/aim2_record_base.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/aim2_record_base.yaml new file mode 100644 index 00000000..e614eb3f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/aim2_record_base.yaml @@ -0,0 +1,60 @@ +id: https://w3id.org/ca_biositing/aim2_records/aim2_record_base +name: aim2_record_base +title: Aim 2 Record Base +description: Base class for Aim 2 records. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - ../core/base_entity + - ../general_analysis/dataset + - ../experiment_equipment/experiment + +classes: + Aim2RecordBase: + is_a: BaseEntity + abstract: true + slots: + - dataset_id + - experiment_id + - resource_id + - sample_id + - technical_replicate_total + - technical_replicate_no + - calc_method_id + - calc_analyst_id + - raw_data_id + - qc_pass + - note + +slots: + experiment_id: + range: integer + description: Reference to Experiment. + resource_id: + range: integer + sample_id: + range: integer + technical_replicate_total: + range: integer + technical_replicate_no: + range: integer + calc_method_id: + range: integer + description: Reference to Method used in calculation. + calc_analyst_id: + range: integer + description: Reference to Analyst who performed calculation. + raw_data_id: + range: integer + description: Reference to FileObjectMetadata. + qc_pass: + range: boolean + description: Whether the record passed QC. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/autoclave_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/autoclave_record.yaml new file mode 100644 index 00000000..d6d96bed --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/autoclave_record.yaml @@ -0,0 +1,22 @@ +id: https://w3id.org/ca_biositing/aim2_records/autoclave_record +name: autoclave_record +title: Autoclave Record +description: Autoclave record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim2_record_base + +classes: + AutoclaveRecord: + is_a: Aim2RecordBase + description: Autoclave record. + slots: [] diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/fermentation_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/fermentation_record.yaml new file mode 100644 index 00000000..f6252706 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/fermentation_record.yaml @@ -0,0 +1,58 @@ +id: https://w3id.org/ca_biositing/aim2_records/fermentation_record +name: fermentation_record +title: Fermentation Record +description: Fermentation record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim2_record_base + - ../methods_parameters_units/method + - ../experiment_equipment/equipment + +classes: + FermentationRecord: + is_a: Aim2RecordBase + description: Fermentation record. + slots: + - strain_id + - pretreatment_method_id + - eh_method_id + - well_position + - temperature + - agitation_rpm + - vessel_id + - analyte_detection_equipment_id + +slots: + strain_id: + range: integer + description: Reference to Strain. + pretreatment_method_id: + range: integer + description: Reference to Method (pretreatment). + eh_method_id: + range: integer + description: Reference to Method (enzymatic hydrolysis). + well_position: + range: string + description: Position in the well. + temperature: + range: decimal + description: Temperature. + agitation_rpm: + range: decimal + description: Agitation in RPM. + vessel_id: + range: integer + description: Reference to Equipment (vessel). + analyte_detection_equipment_id: + range: integer + description: Reference to Equipment (detection). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasifcation_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasifcation_record.yaml new file mode 100644 index 00000000..e69de29b diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasification_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasification_record.yaml new file mode 100644 index 00000000..81bc6cfd --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/gasification_record.yaml @@ -0,0 +1,36 @@ +id: https://w3id.org/ca_biositing/aim2_records/gasification_record +name: gasification_record +title: Gasification Record +description: Gasification record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim2_record_base + +classes: + GasificationRecord: + is_a: Aim2RecordBase + description: Gasification record. + slots: + - feedstock_mass + - bed_temperature + - gas_flow_rate + +slots: + feedstock_mass: + range: decimal + description: Mass of feedstock. + bed_temperature: + range: decimal + description: Bed temperature. + gas_flow_rate: + range: decimal + description: Gas flow rate. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/pretreatment_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/pretreatment_record.yaml new file mode 100644 index 00000000..d4309680 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/pretreatment_record.yaml @@ -0,0 +1,49 @@ +id: https://w3id.org/ca_biositing/aim2_records/pretreatment_record +name: pretreatment_record +title: Pretreatment Record +description: Pretreatment record. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - aim2_record_base + - ../methods_parameters_units/method + +classes: + PretreatmentRecord: + is_a: Aim2RecordBase + description: Pretreatment record. + slots: + - pretreatment_method_id + - eh_method_id + - reaction_block_id + - block_position + - temperature + - replicate_no + +slots: + pretreatment_method_id: + range: integer + description: Reference to Method (pretreatment). + eh_method_id: + range: integer + description: Reference to Method (enzymatic hydrolysis). + reaction_block_id: + range: integer + description: ID of the reaction block. + block_position: + range: string + description: Position in the block. + temperature: + range: decimal + description: Temperature. + replicate_no: + range: integer + description: Replicate number. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/strain.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/strain.yaml new file mode 100644 index 00000000..e16307b7 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/aim2_records/strain.yaml @@ -0,0 +1,27 @@ +id: https://w3id.org/ca_biositing/aim2_records/strain +name: strain +title: Strain +description: Strain used in fermentation. +license: https://creativecommons.org/publicdomain/zero/1.0/ + +prefixes: + linkml: https://w3id.org/linkml/ + ca_biositing: https://w3id.org/ca_biositing/ + +default_prefix: ca_biositing +default_range: string + +imports: + - linkml:types + - ../core/lookup_base + +classes: + Strain: + is_a: LookupBase + description: Strain used in fermentation. + slots: + - parent_strain_id + +slots: + parent_strain_id: + range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core.yaml deleted file mode 100644 index 870eb513..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core.yaml +++ /dev/null @@ -1,79 +0,0 @@ -id: https://w3id.org/ca_biositing/core -name: core -title: Core Definitions -description: Base classes, lineage, and data source definitions. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - -classes: - BaseEntity: - mixin: true - description: Base entity included in all main entity tables. - slots: - - id - - created_at - - updated_at - - etl_run_id - - lineage_group_id - - LookupBase: - mixin: true - description: Base class for enum/ontology-like tables. - slots: - - id - - name - - description - - uri - -slots: - ##Slots from BaseEntity - - id: - identifier: true - range: integer - description: Unique identifier. - - created_at: - range: datetime - description: Timestamp when the record was created. - - updated_at: - range: datetime - description: Timestamp when the record was last updated. - - etl_run_id: - range: string - description: Identifier of the ETL run. - - lineage_group_id: - range: integer - description: Reference to the lineage group. - - ##Slots from LookupBase - - name: - range: string - description: Name of the entity. - - description: - range: string - description: Description of the entity. - - uri: - range: uri - description: URI for the entity. - - ##Slots for other basic fields used across the schema - - note: - range: string - description: Additional notes. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/base_entity.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/base_entity.yaml new file mode 100644 index 00000000..7fcc3d49 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/base_entity.yaml @@ -0,0 +1,40 @@ +id: https://w3id.org/ca_biositing/core/base_entity +name: base_entity +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string + +classes: + BaseEntity: + mixin: true + description: Base entity included in all main entity tables. + slots: + - id + - created_at + - updated_at + - etl_run_id + - lineage_group_id + +slots: + id: + identifier: true + range: integer + description: Unique identifier. + + created_at: + range: datetime + description: Timestamp when the record was created. + + updated_at: + range: datetime + description: Timestamp when the record was last updated. + + etl_run_id: + range: string + description: Identifier of the ETL run. + + lineage_group_id: + range: integer + description: Reference to the lineage group. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/lookup_base.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/lookup_base.yaml new file mode 100644 index 00000000..62932654 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/lookup_base.yaml @@ -0,0 +1,34 @@ +id: https://w3id.org/ca_biositing/core/lookup_base +name: lookup_base +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string + +classes: + LookupBase: + mixin: true + description: Base class for enum/ontology-like tables. + slots: + - id + - name + - description + - uri + +slots: + id: + identifier: true + range: integer + description: Unique identifier. + name: + range: string + description: Name of the entity. + + description: + range: string + description: Description of the entity. + + uri: + range: uri + description: URI for the entity. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/note.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/note.yaml new file mode 100644 index 00000000..91373afd --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/core/note.yaml @@ -0,0 +1,12 @@ +id: https://w3id.org/ca_biositing/core/note +name: note +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string + +slots: + note: + range: string + description: Additional notes. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source.yaml similarity index 55% rename from src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata.yaml rename to src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source.yaml index 1eeb2f53..648926b6 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source.yaml @@ -1,20 +1,14 @@ -id: https://w3id.org/ca_biositing/data_sources_metadata -name: data_sources_metadata -title: Data Sources and Metadata -description: Data sources, types, and file metadata. -license: https://creativecommons.org/publicdomain/zero/1.0/ - +id: https://w3id.org/ca_biositing/data_sources_metadata/data_source +name: data_source prefixes: linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - imports: - linkml:types - - core - - field_sampling + - ../core/base_entity + - data_source_type + - location_resolution + - ../core/note +default_range: string classes: DataSource: @@ -42,68 +36,9 @@ classes: - uri - note - FileObjectMetadata: - is_a: BaseEntity - description: Metadata for a file object. - slots: - - data_source_id - - bucket_path - - file_format - - file_size - - checksum_md5 - - checksum_sha256 - - DataSourceType: - is_a: BaseEntity - description: Type of data source (e.g. database, literature). - slots: - - source_type_id - - LocationResolution: - is_a: LookupBase - description: Resolution of the location (e.g. nation, state, county). - - SourceType: - is_a: LookupBase - description: Type of source (e.g. database, literature). - slots: - #FileObjectMetadata slots - - data_source_id: - range: integer - description: Reference to DataSource. - - bucket_path: - range: string - description: Path to the file in the bucket. - - file_format: - range: string - description: Format of the file. - - file_size: - range: integer - description: Size of the file in bytes. - - checksum_md5: - range: string - description: MD5 checksum of the file. - - checksum_sha256: - range: string - description: SHA256 checksum of the file. - - #DataSourceType slots - - source_type_id: - range: integer - description: Reference to SourceType. - - #DataSource slots - data_source_type_id: - range: integer + range: DataSourceType description: Reference to DataSourceType. full_title: @@ -157,7 +92,7 @@ slots: description: The temporal topic of the resource (Daterange). location_coverage_id: - range: integer + range: LocationResolution description: Reference to LocationResolution. rights: diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source_type.yaml new file mode 100644 index 00000000..924d9e5f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/data_source_type.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/data_sources_metadata/data_source_type +name: data_source_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - source_type +default_range: string + +classes: + DataSourceType: + is_a: BaseEntity + description: Type of data source (e.g. database, literature). + slots: + - source_type_id + +slots: + source_type_id: + range: SourceType + description: Reference to SourceType. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/file_object_metadata.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/file_object_metadata.yaml new file mode 100644 index 00000000..919b6ca1 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/file_object_metadata.yaml @@ -0,0 +1,46 @@ +id: https://w3id.org/ca_biositing/data_sources_metadata/file_object_metadata +name: file_object_metadata +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - data_source +default_range: string + +classes: + FileObjectMetadata: + is_a: BaseEntity + description: Metadata for a file object. + slots: + - data_source_id + - bucket_path + - file_format + - file_size + - checksum_md5 + - checksum_sha256 + +slots: + data_source_id: + range: DataSource + description: Reference to DataSource. + + bucket_path: + range: string + description: Path to the file in the bucket. + + file_format: + range: string + description: Format of the file. + + file_size: + range: integer + description: Size of the file in bytes. + + checksum_md5: + range: string + description: MD5 checksum of the file. + + checksum_sha256: + range: string + description: SHA256 checksum of the file. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/location_resolution.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/location_resolution.yaml new file mode 100644 index 00000000..89c3ccdc --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/location_resolution.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/data_sources_metadata/location_resolution +name: location_resolution +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + LocationResolution: + is_a: LookupBase + description: Resolution of the location (e.g. nation, state, county). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/source_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/source_type.yaml new file mode 100644 index 00000000..f4ad79dc --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/data_sources_metadata/source_type.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/data_sources_metadata/source_type +name: source_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + SourceType: + is_a: LookupBase + description: Type of source (e.g. database, literature). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment.yaml deleted file mode 100644 index bc76394b..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment.yaml +++ /dev/null @@ -1,127 +0,0 @@ -id: https://w3id.org/ca_biositing/experiment_equipment -name: experiment_equipment -title: Experiments and Equipment -description: Experimental data and equipment. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - places - - methods_parameters_units - - sample_preparation - - general_analysis - -classes: - Experiment: - is_a: BaseEntity - description: Experiment definition. - slots: - - analyst_id - - exper_start_date - - exper_duration - - exper_duration_unit_id - - exper_location_id - - description - - Equipment: - is_a: LookupBase - description: Equipment used in experiments. - slots: - - equipment_location_id - - ExperimentMethod: - description: Link between Experiment and Method. - slots: - - id - - experiment_id - - method_id - slot_usage: - id: - identifier: true - range: integer - - ExperimentEquipment: - description: Link between Experiment and Equipment. - slots: - - id - - experiment_id - - equipment_id - slot_usage: - id: - identifier: true - range: integer - - ExperimentAnalysis: - description: Link between Experiment and AnalysisType. - slots: - - id - - experiment_id - - analysis_type_id - slot_usage: - id: - identifier: true - range: integer - - ExperimentPreparedSample: - description: Link between Experiment and PreparedSample. - slots: - - id - - experiment_id - - prepared_sample_id - slot_usage: - id: - identifier: true - range: integer - -slots: - analyst_id: - range: integer - description: Reference to Contact (analyst). - - exper_start_date: - range: date - description: Start date of the experiment. - - exper_duration: - range: decimal - description: Duration of the experiment. - - exper_duration_unit_id: - range: integer - description: Reference to Unit. - - exper_location_id: - range: integer - description: Reference to LocationAddress. - - equipment_location_id: - range: integer - description: Reference to LocationAddress. - - experiment_id: - range: integer - description: Reference to Experiment. - - equipment_id: - range: integer - description: Reference to Equipment. - - analysis_type_id: - range: integer - description: Reference to AnalysisType. - - prepared_sample_id: - range: integer - description: Reference to PreparedSample. - - method_id: - range: integer - description: Reference to Method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/equipment.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/equipment.yaml new file mode 100644 index 00000000..fb403b50 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/equipment.yaml @@ -0,0 +1,21 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/equipment +name: equipment +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base + - ../places/location_address +default_range: string + +classes: + Equipment: + is_a: LookupBase + description: Equipment used in experiments. + slots: + - equipment_location_id + +slots: + equipment_location_id: + range: LocationAddress + description: Reference to LocationAddress. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment.yaml new file mode 100644 index 00000000..3719fea4 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment.yaml @@ -0,0 +1,44 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/experiment +name: experiment +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../people/contact + - ../places/location_address + - ../methods_parameters_units/unit +default_range: string + +classes: + Experiment: + is_a: BaseEntity + description: Experiment definition. + slots: + - analyst_id + - exper_start_date + - exper_duration + - exper_duration_unit_id + - exper_location_id + - description + +slots: + analyst_id: + range: Contact + description: Reference to Contact (analyst). + + exper_start_date: + range: date + description: Start date of the experiment. + + exper_duration: + range: decimal + description: Duration of the experiment. + + exper_duration_unit_id: + range: Unit + description: Reference to Unit. + + exper_location_id: + range: LocationAddress + description: Reference to LocationAddress. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_analysis.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_analysis.yaml new file mode 100644 index 00000000..f6be8596 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_analysis.yaml @@ -0,0 +1,30 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/experiment_analysis +name: experiment_analysis +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - experiment + - ../general_analysis/analysis_type +default_range: string + +classes: + ExperimentAnalysis: + description: Link between Experiment and AnalysisType. + slots: + - id + - experiment_id + - analysis_type_id + slot_usage: + id: + identifier: true + range: integer + +slots: + experiment_id: + range: Experiment + description: Reference to Experiment. + + analysis_type_id: + range: AnalysisType + description: Reference to AnalysisType. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_equipment.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_equipment.yaml new file mode 100644 index 00000000..7197ebeb --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_equipment.yaml @@ -0,0 +1,30 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/experiment_equipment +name: experiment_equipment +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - experiment + - equipment +default_range: string + +classes: + ExperimentEquipment: + description: Link between Experiment and Equipment. + slots: + - id + - experiment_id + - equipment_id + slot_usage: + id: + identifier: true + range: integer + +slots: + experiment_id: + range: Experiment + description: Reference to Experiment. + + equipment_id: + range: Equipment + description: Reference to Equipment. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_method.yaml new file mode 100644 index 00000000..e0a9e35d --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_method.yaml @@ -0,0 +1,31 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/experiment_method +name: experiment_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - experiment + - ../methods_parameters_units/method +default_range: string + +classes: + ExperimentMethod: + description: Link between Experiment and Method. + slots: + - id + - experiment_id + - method_id + slot_usage: + id: + identifier: true + range: integer + +slots: + experiment_id: + range: Experiment + description: Reference to Experiment. + + method_id: + range: Method + description: Reference to Method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_prepared_sample.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_prepared_sample.yaml new file mode 100644 index 00000000..969e235f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/experiment_equipment/experiment_prepared_sample.yaml @@ -0,0 +1,31 @@ +id: https://w3id.org/ca_biositing/experiment_equipment/experiment_prepared_sample +name: experiment_prepared_sample +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - experiment + - ../sample_preparation/prepared_sample +default_range: string + +classes: + ExperimentPreparedSample: + description: Link between Experiment and PreparedSample. + slots: + - id + - experiment_id + - prepared_sample_id + slot_usage: + id: + identifier: true + range: integer + +slots: + experiment_id: + range: Experiment + description: Reference to Experiment. + + prepared_sample_id: + range: PreparedSample + description: Reference to PreparedSample. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml deleted file mode 100644 index 679a5f4a..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data.yaml +++ /dev/null @@ -1,441 +0,0 @@ -id: https://w3id.org/ca_biositing/external_data -name: external_data -title: External Data -description: External datasets (USDA, LandIQ, etc.). -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - general_analysis - - places - - resource_information - - methods_parameters_units - - data_sources_metadata - -classes: - UsdaDomain: - is_a: LookupBase - description: USDA domain. - - UsdaStatisticCategory: - is_a: LookupBase - description: USDA statistic category. - - UsdaSurveyProgram: - is_a: LookupBase - description: USDA survey program. - - UsdaCensusRecord: - is_a: BaseEntity - description: USDA census record. - slots: - - dataset_id - - geoid - - commodity_code - - year - - source_reference - - note - - UsdaSurveyRecord: - is_a: BaseEntity - description: USDA survey record. - slots: - - dataset_id - - geoid - - commodity_code - - year - - survey_program_id - - survey_period - - reference_month - - seasonal_flag - - note - - #RESOURCE ONTOLOGY MAPPING - - UsdaCommodity: - is_a: LookupBase - description: USDA commodity. - slots: - - usda_source - - usda_code - - parent_commodity_id - - UsdaTermMap: - is_a: BaseEntity - description: Mapping of raw terms to USDA commodities. - slots: - - source_system - - source_context - - raw_term - - usda_commodity_id - - is_verified - - note - - ResourceUsdaCommodityMap: - is_a: BaseEntity - description: Mapping between resources/crops and USDA commodities. - slots: - - resource_id - - primary_ag_product_id - - usda_commodity_id - - match_tier - - note - - #USDA MARKET DATA - - UsdaMarketReport: - is_a: BaseEntity - description: USDA market report. - slots: - - slug_id - - slug_name - - report_series_title - - frequency - - office_name - - office_city_id - - office_state_fips - - source_id - - UsdaMarketRecord: - is_a: BaseEntity - description: USDA market record. - slots: - - report_id - - dataset_id - - report_begin_date - - report_end_date - - report_date - - commodity_id - - market_type_id - - market_type_category - - grp - - market_category_id - - class_ - - grade - - variety - - protein_pct - - application - - pkg - - sale_type - - price_unit_id - - freight - - trans_mode - - #LANDIQ DATA - - LandiqRecord: - is_a: BaseEntity - description: LandIQ record. - slots: - - dataset_id - - polygon_id - - main_crop - - secondary_crop - - tertiary_crop - - quaternary_crop - - confidence - - irrigated - - acres - - version - - note - - Polygon: - description: Geospatial polygon. - slots: - - id - - geoid - - geom - slot_usage: - id: - identifier: true - range: integer - - #BILLION TON DATA - - BillionTon2023Record: - is_a: BaseEntity - description: Billion Ton 2023 record. - slots: - - subclass_id - - resource_id - - geoid - - county_square_miles - - model_name - - scenario_name - - price_offered_usd - - production - - production_unit_id - - btu_ton - - production_energy_content - - energy_content_unit_id - - product_density_dtpersqmi - - land_source - -slots: - usda_source: - range: string - description: Source of the USDA definition (NASS, AMS). - - usda_code: - range: string - description: Official USDA code. - - parent_commodity_id: - range: integer - description: Reference to parent UsdaCommodity. - - source_system: - range: string - description: Source system (NASS, AMS). - - source_context: - range: string - description: Context of the source (Group, Slug ID). - - raw_term: - range: string - description: Raw term from the source. - - usda_commodity_id: - range: integer - description: Reference to UsdaCommodity. - - is_verified: - range: boolean - description: Whether the mapping is verified. - - match_tier: - range: string - description: Tier of the match. - - commodity_code: - range: integer - description: Reference to UsdaCommodity. - - year: - range: integer - description: Year of the record. - - source_reference: - range: string - description: Reference to the source. - - survey_program_id: - range: integer - description: Reference to UsdaSurveyProgram. - - survey_period: - range: string - description: Period of the survey. - - reference_month: - range: string - description: Reference month. - - seasonal_flag: - range: boolean - description: Whether the record is seasonal. - - slug_id: - range: integer - description: Slug ID. - - slug_name: - range: string - description: Slug name. - - report_series_title: - range: string - description: Title of the report series. - - frequency: - range: string - description: Frequency of the report. - - office_name: - range: string - description: Name of the office. - - office_city_id: - range: integer - description: Reference to LocationAddress. - - office_state_fips: - range: string - description: FIPS code of the office state. - - report_id: - range: integer - description: Reference to UsdaMarketReport. - - report_begin_date: - range: datetime - description: Begin date of the report. - - report_end_date: - range: datetime - description: End date of the report. - - report_date: - range: datetime - description: Date of the report. - - commodity_id: - range: integer - description: Reference to UsdaCommodity. - - market_type_id: - range: integer - description: ID of the market type. - - market_type_category: - range: string - description: Category of the market type. - - grp: - range: string - description: Group. - - market_category_id: - range: integer - description: ID of the market category. - - class_: - range: string - description: Class. - - grade: - range: string - description: Grade. - - variety: - range: string - description: Variety. - - protein_pct: - range: decimal - description: Protein percentage. - - application: - range: string - description: Application. - - pkg: - range: string - description: Package. - - sale_type: - range: string - description: Sale type. - - price_unit_id: - range: integer - description: Reference to Unit. - - freight: - range: string - description: Freight. - - trans_mode: - range: string - description: Transportation mode. - - polygon_id: - range: integer - description: Reference to Polygon. - - main_crop: - range: integer - description: Reference to PrimaryAgProduct. - - secondary_crop: - range: integer - description: Reference to PrimaryAgProduct. - - tertiary_crop: - range: integer - description: Reference to PrimaryAgProduct. - - quaternary_crop: - range: integer - description: Reference to PrimaryAgProduct. - - confidence: - range: integer - description: Confidence level. - - irrigated: - range: boolean - description: Whether the land is irrigated. - - acres: - range: float - description: Number of acres. - - version: - range: string - description: Version of the data source. - - subclass_id: - range: integer - description: Reference to ResourceSubclass. - - county_square_miles: - range: float - description: Square miles of the county. - - model_name: - range: string - description: Name of the model. - - scenario_name: - range: string - description: Name of the scenario. - - price_offered_usd: - range: decimal - description: Price offered in USD. - - production: - range: integer - description: Production amount. - - production_unit_id: - range: integer - description: Reference to Unit. - - btu_ton: - range: integer - description: BTU per ton. - - production_energy_content: - range: integer - description: Energy content of production. - - energy_content_unit_id: - range: integer - description: Reference to Unit. - - product_density_dtpersqmi: - range: decimal - description: Product density in dry tons per square mile. - - land_source: - range: string - description: Source of land data. - - geom: - range: string - description: Geometry (WKT or similar). - - test: - range: string - description: A test slot. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/billion_ton_2023_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/billion_ton_2023_record.yaml new file mode 100644 index 00000000..24e9e470 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/billion_ton_2023_record.yaml @@ -0,0 +1,89 @@ +id: https://w3id.org/ca_biositing/external_data/billion_ton_2023_record +name: billion_ton_2023_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../resource_information/resource + - ../resource_information/resource_subclass + - ../places/geography + - ../methods_parameters_units/unit +default_range: string + +classes: + BillionTon2023Record: + is_a: BaseEntity + description: Billion Ton 2023 record. + slots: + - subclass_id + - resource_id + - geoid + - county_square_miles + - model_name + - scenario_name + - price_offered_usd + - production + - production_unit_id + - btu_ton + - production_energy_content + - energy_content_unit_id + - product_density_dtpersqmi + - land_source + +slots: + subclass_id: + range: ResourceSubclass + description: Reference to ResourceSubclass. + + resource_id: + range: Resource + description: Reference to Resource. + + geoid: + range: Geography + description: Reference to Geography. + + county_square_miles: + range: float + description: Square miles of the county. + + model_name: + range: string + description: Name of the model. + + scenario_name: + range: string + description: Name of the scenario. + + price_offered_usd: + range: decimal + description: Price offered in USD. + + production: + range: integer + description: Production amount. + + production_unit_id: + range: Unit + description: Reference to Unit. + + btu_ton: + range: integer + description: BTU per ton. + + production_energy_content: + range: integer + description: Energy content of production. + + energy_content_unit_id: + range: Unit + description: Reference to Unit. + + product_density_dtpersqmi: + range: decimal + description: Product density in dry tons per square mile. + + land_source: + range: string + description: Source of land data. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/landiq_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/landiq_record.yaml new file mode 100644 index 00000000..1827575b --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/landiq_record.yaml @@ -0,0 +1,69 @@ +id: https://w3id.org/ca_biositing/external_data/landiq_record +name: landiq_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../data_sources_metadata/data_source + - polygon + - ../resource_information/primary_ag_product +default_range: string + +classes: + LandiqRecord: + is_a: BaseEntity + description: LandIQ record. + slots: + - dataset_id + - polygon_id + - main_crop + - secondary_crop + - tertiary_crop + - quaternary_crop + - confidence + - irrigated + - acres + - version + - note + +slots: + dataset_id: + range: DataSource + description: Reference to DataSource. + + polygon_id: + range: Polygon + description: Reference to Polygon. + + main_crop: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + secondary_crop: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + tertiary_crop: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + quaternary_crop: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + confidence: + range: integer + description: Confidence level. + + irrigated: + range: boolean + description: Whether the land is irrigated. + + acres: + range: float + description: Number of acres. + + version: + range: string + description: Version of the data source. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/polygon.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/polygon.yaml new file mode 100644 index 00000000..5af8c173 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/polygon.yaml @@ -0,0 +1,30 @@ +id: https://w3id.org/ca_biositing/external_data/polygon +name: polygon +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../places/geography +default_range: string + +classes: + Polygon: + description: Geospatial polygon. + slots: + - id + - geoid + - geom + slot_usage: + id: + identifier: true + range: integer + +slots: + geoid: + range: Geography + description: Reference to Geography. + + geom: + range: string + description: Geometry (WKT or similar). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/resource_usda_commodity_map.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/resource_usda_commodity_map.yaml new file mode 100644 index 00000000..68ed2726 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/resource_usda_commodity_map.yaml @@ -0,0 +1,39 @@ +id: https://w3id.org/ca_biositing/external_data/resource_usda_commodity_map +name: resource_usda_commodity_map +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../resource_information/resource + - ../resource_information/primary_ag_product + - usda_commodity +default_range: string + +classes: + ResourceUsdaCommodityMap: + is_a: BaseEntity + description: Mapping between resources/crops and USDA commodities. + slots: + - resource_id + - primary_ag_product_id + - usda_commodity_id + - match_tier + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. + + primary_ag_product_id: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + usda_commodity_id: + range: UsdaCommodity + description: Reference to UsdaCommodity. + + match_tier: + range: string + description: Tier of the match. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_census_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_census_record.yaml new file mode 100644 index 00000000..0760d9cd --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_census_record.yaml @@ -0,0 +1,44 @@ +id: https://w3id.org/ca_biositing/external_data/usda_census_record +name: usda_census_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../data_sources_metadata/data_source + - ../places/geography + - usda_commodity +default_range: string + +classes: + UsdaCensusRecord: + is_a: BaseEntity + description: USDA census record. + slots: + - dataset_id + - geoid + - commodity_code + - year + - source_reference + - note + +slots: + dataset_id: + range: DataSource + description: Reference to DataSource. + + geoid: + range: Geography + description: Reference to Geography. + + commodity_code: + range: UsdaCommodity + description: Reference to UsdaCommodity. + + year: + range: integer + description: Year of the record. + + source_reference: + range: string + description: Reference to the source. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_commodity.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_commodity.yaml new file mode 100644 index 00000000..4bb95641 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_commodity.yaml @@ -0,0 +1,30 @@ +id: https://w3id.org/ca_biositing/external_data/usda_commodity +name: usda_commodity +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + UsdaCommodity: + is_a: LookupBase + description: USDA commodity. + slots: + - usda_source + - usda_code + - parent_commodity_id + +slots: + usda_source: + range: string + description: Source of the USDA definition (NASS, AMS). + + usda_code: + range: string + description: Official USDA code. + + parent_commodity_id: + range: UsdaCommodity + description: Reference to parent UsdaCommodity. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_domain.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_domain.yaml new file mode 100644 index 00000000..6e12bb15 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_domain.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/external_data/usda_domain +name: usda_domain +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + UsdaDomain: + is_a: LookupBase + description: USDA domain. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_record.yaml new file mode 100644 index 00000000..c128596a --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_record.yaml @@ -0,0 +1,119 @@ +id: https://w3id.org/ca_biositing/external_data/usda_market_record +name: usda_market_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - usda_market_report + - ../data_sources_metadata/data_source + - usda_commodity + - ../methods_parameters_units/unit +default_range: string + +classes: + UsdaMarketRecord: + is_a: BaseEntity + description: USDA market record. + slots: + - report_id + - dataset_id + - report_begin_date + - report_end_date + - report_date + - commodity_id + - market_type_id + - market_type_category + - grp + - market_category_id + - class_ + - grade + - variety + - protein_pct + - application + - pkg + - sale_type + - price_unit_id + - freight + - trans_mode + +slots: + report_id: + range: UsdaMarketReport + description: Reference to UsdaMarketReport. + + dataset_id: + range: DataSource + description: Reference to DataSource. + + report_begin_date: + range: datetime + description: Begin date of the report. + + report_end_date: + range: datetime + description: End date of the report. + + report_date: + range: datetime + description: Date of the report. + + commodity_id: + range: UsdaCommodity + description: Reference to UsdaCommodity. + + market_type_id: + range: integer + description: ID of the market type. + + market_type_category: + range: string + description: Category of the market type. + + grp: + range: string + description: Group. + + market_category_id: + range: integer + description: ID of the market category. + + class_: + range: string + description: Class. + + grade: + range: string + description: Grade. + + variety: + range: string + description: Variety. + + protein_pct: + range: decimal + description: Protein percentage. + + application: + range: string + description: Application. + + pkg: + range: string + description: Package. + + sale_type: + range: string + description: Sale type. + + price_unit_id: + range: Unit + description: Reference to Unit. + + freight: + range: string + description: Freight. + + trans_mode: + range: string + description: Transportation mode. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_report.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_report.yaml new file mode 100644 index 00000000..f5254218 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_market_report.yaml @@ -0,0 +1,57 @@ +id: https://w3id.org/ca_biositing/external_data/usda_market_report +name: usda_market_report +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../places/location_address + - ../data_sources_metadata/data_source +default_range: string + +classes: + UsdaMarketReport: + is_a: BaseEntity + description: USDA market report. + slots: + - slug_id + - slug_name + - report_series_title + - frequency + - office_name + - office_city_id + - office_state_fips + - source_id + +slots: + slug_id: + range: integer + description: Slug ID. + + slug_name: + range: string + description: Slug name. + + report_series_title: + range: string + description: Title of the report series. + + frequency: + range: string + description: Frequency of the report. + + office_name: + range: string + description: Name of the office. + + office_city_id: + range: LocationAddress + description: Reference to LocationAddress. + + office_state_fips: + range: string + description: FIPS code of the office state. + + source_id: + range: DataSource + description: Reference to DataSource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_statistic_category.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_statistic_category.yaml new file mode 100644 index 00000000..5a410d79 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_statistic_category.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/external_data/usda_statistic_category +name: usda_statistic_category +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + UsdaStatisticCategory: + is_a: LookupBase + description: USDA statistic category. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_program.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_program.yaml new file mode 100644 index 00000000..1933375d --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_program.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/external_data/usda_survey_program +name: usda_survey_program +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + UsdaSurveyProgram: + is_a: LookupBase + description: USDA survey program. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_record.yaml new file mode 100644 index 00000000..653ea194 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_survey_record.yaml @@ -0,0 +1,60 @@ +id: https://w3id.org/ca_biositing/external_data/usda_survey_record +name: usda_survey_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../data_sources_metadata/data_source + - ../places/geography + - usda_commodity + - usda_survey_program +default_range: string + +classes: + UsdaSurveyRecord: + is_a: BaseEntity + description: USDA survey record. + slots: + - dataset_id + - geoid + - commodity_code + - year + - survey_program_id + - survey_period + - reference_month + - seasonal_flag + - note + +slots: + dataset_id: + range: DataSource + description: Reference to DataSource. + + geoid: + range: Geography + description: Reference to Geography. + + commodity_code: + range: UsdaCommodity + description: Reference to UsdaCommodity. + + year: + range: integer + description: Year of the record. + + survey_program_id: + range: UsdaSurveyProgram + description: Reference to UsdaSurveyProgram. + + survey_period: + range: string + description: Period of the survey. + + reference_month: + range: string + description: Reference month. + + seasonal_flag: + range: boolean + description: Whether the record is seasonal. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_term_map.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_term_map.yaml new file mode 100644 index 00000000..17711897 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/external_data/usda_term_map.yaml @@ -0,0 +1,42 @@ +id: https://w3id.org/ca_biositing/external_data/usda_term_map +name: usda_term_map +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - usda_commodity +default_range: string + +classes: + UsdaTermMap: + is_a: BaseEntity + description: Mapping of raw terms to USDA commodities. + slots: + - source_system + - source_context + - raw_term + - usda_commodity_id + - is_verified + - note + +slots: + source_system: + range: string + description: Source system (NASS, AMS). + + source_context: + range: string + description: Context of the source (Group, Slug ID). + + raw_term: + range: string + description: Raw term from the source. + + usda_commodity_id: + range: UsdaCommodity + description: Reference to UsdaCommodity. + + is_verified: + range: boolean + description: Whether the mapping is verified. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling.yaml deleted file mode 100644 index 74ab00d6..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling.yaml +++ /dev/null @@ -1,200 +0,0 @@ -id: https://w3id.org/ca_biositing/field_sampling -name: field_sampling -title: Field Sampling -description: Field sampling and preparation. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - places - - people - - resource_information - - methods_parameters_units - -classes: - FieldSample: - is_a: BaseEntity - description: Sample collected from the field. - slots: - - name - - resource_id - - provider_id - - collector_id - - sample_collection_source - - amount_collected - - amount_collected_unit_id - - sampling_location_id - - field_storage_method_id - - field_storage_duration_value - - field_storage_duration_unit_id - - field_storage_location_id - - collection_timestamp - - collection_method_id - - harvest_method_id - - harvest_date - - field_sample_storage_location_id - - note - - PhysicalCharacteristic: - is_a: BaseEntity - description: Physical characteristics of a sample. - slots: - - field_sample_id - - particle_length - - particle_width - - particle_height - - particle_unit_id - - FieldSampleCondition: - is_a: BaseEntity - description: Condition of the field sample. - slots: - - field_sample_id - - ag_treatment_id - - last_application_date - - treatment_amount_per_acre - - processing_method_id - - FieldStorageMethod: - is_a: LookupBase - description: Method of field storage. - - CollectionMethod: - is_a: LookupBase - description: Method of collection. - - HarvestMethod: - is_a: LookupBase - description: Method of harvest. - - ProcessingMethod: - is_a: LookupBase - description: Method of processing. - - SoilType: - is_a: LookupBase - description: Type of soil. - - AgTreatment: - is_a: LookupBase - description: Agricultural treatment. - - LocationSoilType: - is_a: BaseEntity - description: Soil type at a location. - slots: - - location_id - - soil_type_id - -slots: - provider_id: - range: integer - description: Reference to Provider. - - collector_id: - range: integer - description: Reference to Contact (collector). - - sample_collection_source: - range: string - description: Source of sample collection. - - amount_collected: - range: decimal - description: Amount collected. - - amount_collected_unit_id: - range: integer - description: Reference to Unit. - - sampling_location_id: - range: integer - description: Reference to LocationAddress. - - field_storage_method_id: - range: integer - description: Reference to FieldStorageMethod. - - field_storage_duration_value: - range: decimal - description: Duration of field storage. - - field_storage_duration_unit_id: - range: integer - description: Reference to Unit. - - field_storage_location_id: - range: integer - description: Reference to LocationAddress. - - collection_timestamp: - range: datetime - description: Timestamp of collection. - - collection_method_id: - range: integer - description: Reference to CollectionMethod. - - harvest_method_id: - range: integer - description: Reference to HarvestMethod. - - harvest_date: - range: date - description: Date of harvest. - - field_sample_storage_location_id: - range: integer - description: Reference to LocationAddress. - - field_sample_id: - range: integer - description: Reference to FieldSample. - - particle_length: - range: decimal - description: Length of particle. - - particle_width: - range: decimal - description: Width of particle. - - particle_height: - range: decimal - description: Height of particle. - - particle_unit_id: - range: integer - description: Reference to Unit. - - ag_treatment_id: - range: integer - description: Reference to AgTreatment. - - last_application_date: - range: date - description: Date of last application. - - treatment_amount_per_acre: - range: float - description: Amount of treatment per acre. - - processing_method_id: - range: integer - description: Reference to ProcessingMethod. - - location_id: - range: integer - description: Reference to LocationAddress. - - soil_type_id: - range: integer - description: Reference to SoilType. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/ag_treatment.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/ag_treatment.yaml new file mode 100644 index 00000000..1edf4f87 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/ag_treatment.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/ag_treatment +name: ag_treatment +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + AgTreatment: + is_a: LookupBase + description: Agricultural treatment. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/collection_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/collection_method.yaml new file mode 100644 index 00000000..3aa95cc5 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/collection_method.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/collection_method +name: collection_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + CollectionMethod: + is_a: LookupBase + description: Method of collection. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample.yaml new file mode 100644 index 00000000..4f520bf6 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample.yaml @@ -0,0 +1,105 @@ +id: https://w3id.org/ca_biositing/field_sampling/field_sample +name: field_sample +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../places/location_address + - ../people/contact + - ../people/provider + - ../resource_information/resource + - ../methods_parameters_units/unit + - field_storage_method + - collection_method + - harvest_method +default_range: string + +classes: + FieldSample: + is_a: BaseEntity + description: Sample collected from the field. + slots: + - name + - resource_id + - provider_id + - collector_id + - sample_collection_source + - amount_collected + - amount_collected_unit_id + - sampling_location_id + - field_storage_method_id + - field_storage_duration_value + - field_storage_duration_unit_id + - field_storage_location_id + - collection_timestamp + - collection_method_id + - harvest_method_id + - harvest_date + - field_sample_storage_location_id + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. + + provider_id: + range: Provider + description: Reference to Provider. + + collector_id: + range: Contact + description: Reference to Contact (collector). + + sample_collection_source: + range: string + description: Source of sample collection. + + amount_collected: + range: decimal + description: Amount collected. + + amount_collected_unit_id: + range: Unit + description: Reference to Unit. + + sampling_location_id: + range: LocationAddress + description: Reference to LocationAddress. + + field_storage_method_id: + range: FieldStorageMethod + description: Reference to FieldStorageMethod. + + field_storage_duration_value: + range: decimal + description: Duration of field storage. + + field_storage_duration_unit_id: + range: Unit + description: Reference to Unit. + + field_storage_location_id: + range: LocationAddress + description: Reference to LocationAddress. + + collection_timestamp: + range: datetime + description: Timestamp of collection. + + collection_method_id: + range: CollectionMethod + description: Reference to CollectionMethod. + + harvest_method_id: + range: HarvestMethod + description: Reference to HarvestMethod. + + harvest_date: + range: date + description: Date of harvest. + + field_sample_storage_location_id: + range: LocationAddress + description: Reference to LocationAddress. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample_condition.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample_condition.yaml new file mode 100644 index 00000000..37ed41b0 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_sample_condition.yaml @@ -0,0 +1,43 @@ +id: https://w3id.org/ca_biositing/field_sampling/field_sample_condition +name: field_sample_condition +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - field_sample + - ag_treatment + - processing_method +default_range: string + +classes: + FieldSampleCondition: + is_a: BaseEntity + description: Condition of the field sample. + slots: + - field_sample_id + - ag_treatment_id + - last_application_date + - treatment_amount_per_acre + - processing_method_id + +slots: + field_sample_id: + range: FieldSample + description: Reference to FieldSample. + + ag_treatment_id: + range: AgTreatment + description: Reference to AgTreatment. + + last_application_date: + range: date + description: Date of last application. + + treatment_amount_per_acre: + range: float + description: Amount of treatment per acre. + + processing_method_id: + range: ProcessingMethod + description: Reference to ProcessingMethod. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_storage_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_storage_method.yaml new file mode 100644 index 00000000..9eeebd9b --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/field_storage_method.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/field_storage_method +name: field_storage_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + FieldStorageMethod: + is_a: LookupBase + description: Method of field storage. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/harvest_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/harvest_method.yaml new file mode 100644 index 00000000..4ec880db --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/harvest_method.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/harvest_method +name: harvest_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + HarvestMethod: + is_a: LookupBase + description: Method of harvest. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/location_soil_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/location_soil_type.yaml new file mode 100644 index 00000000..584486c8 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/location_soil_type.yaml @@ -0,0 +1,27 @@ +id: https://w3id.org/ca_biositing/field_sampling/location_soil_type +name: location_soil_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../places/location_address + - soil_type +default_range: string + +classes: + LocationSoilType: + is_a: BaseEntity + description: Soil type at a location. + slots: + - location_id + - soil_type_id + +slots: + location_id: + range: LocationAddress + description: Reference to LocationAddress. + + soil_type_id: + range: SoilType + description: Reference to SoilType. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/physical_characteristic.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/physical_characteristic.yaml new file mode 100644 index 00000000..6644bbb9 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/physical_characteristic.yaml @@ -0,0 +1,42 @@ +id: https://w3id.org/ca_biositing/field_sampling/physical_characteristic +name: physical_characteristic +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - field_sample + - ../methods_parameters_units/unit +default_range: string + +classes: + PhysicalCharacteristic: + is_a: BaseEntity + description: Physical characteristics of a sample. + slots: + - field_sample_id + - particle_length + - particle_width + - particle_height + - particle_unit_id + +slots: + field_sample_id: + range: FieldSample + description: Reference to FieldSample. + + particle_length: + range: decimal + description: Length of particle. + + particle_width: + range: decimal + description: Width of particle. + + particle_height: + range: decimal + description: Height of particle. + + particle_unit_id: + range: Unit + description: Reference to Unit. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/processing_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/processing_method.yaml new file mode 100644 index 00000000..603c4842 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/processing_method.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/processing_method +name: processing_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + ProcessingMethod: + is_a: LookupBase + description: Method of processing. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/soil_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/soil_type.yaml new file mode 100644 index 00000000..b232bc63 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/field_sampling/soil_type.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/field_sampling/soil_type +name: soil_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + SoilType: + is_a: LookupBase + description: Type of soil. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis.yaml deleted file mode 100644 index 6f9db701..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis.yaml +++ /dev/null @@ -1,97 +0,0 @@ -id: https://w3id.org/ca_biositing/general_analysis -name: general_analysis -title: General Analysis -description: General analysis definitions. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - data_sources_metadata - - methods_parameters_units - -classes: - AnalysisType: - is_a: LookupBase - description: Type of analysis. - - Dataset: - is_a: BaseEntity - description: Dataset definition. - slots: - - name - - record_type - - source_id - - start_date - - end_date - - description - - DimensionType: - is_a: LookupBase - description: Type of dimension. - - Observation: - is_a: BaseEntity - description: Observation data. - slots: - - dataset_id - - record_type - - record_id - - parameter_id - - value - - unit_id - - dimension_type_id - - dimension_value - - dimension_unit_id - - note - -slots: - #Slots imported - #parameter_id: from methods.yaml - #note: from core.yaml - - record_type: - range: string - description: Type of record. - - start_date: - range: date - description: Start date of the dataset. - - end_date: - range: date - description: End date of the dataset. - - dataset_id: - range: integer - description: Reference to Dataset. - - record_id: - range: integer - description: ID of the record. - - value: - range: decimal - description: Value of the observation. - - unit_id: - range: integer - description: Reference to Unit. - - dimension_type_id: - range: integer - description: Reference to DimensionType. - - dimension_value: - range: decimal - description: Value of the dimension. - - dimension_unit_id: - range: integer diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/analysis_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/analysis_type.yaml new file mode 100644 index 00000000..6f84aff3 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/analysis_type.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/general_analysis/analysis_type +name: analysis_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + AnalysisType: + is_a: LookupBase + description: Type of analysis. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dataset.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dataset.yaml new file mode 100644 index 00000000..8d3d7550 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dataset.yaml @@ -0,0 +1,38 @@ +id: https://w3id.org/ca_biositing/general_analysis/dataset +name: dataset +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../data_sources_metadata/data_source +default_range: string + +classes: + Dataset: + is_a: BaseEntity + description: Dataset definition. + slots: + - name + - record_type + - source_id + - start_date + - end_date + - description + +slots: + record_type: + range: string + description: Type of record. + + source_id: + range: DataSource + description: Reference to DataSource. + + start_date: + range: date + description: Start date of the dataset. + + end_date: + range: date + description: End date of the dataset. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dimension_type.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dimension_type.yaml new file mode 100644 index 00000000..f92b6838 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/dimension_type.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/general_analysis/dimension_type +name: dimension_type +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + DimensionType: + is_a: LookupBase + description: Type of dimension. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/observation.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/observation.yaml new file mode 100644 index 00000000..f990843c --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/general_analysis/observation.yaml @@ -0,0 +1,64 @@ +id: https://w3id.org/ca_biositing/general_analysis/observation +name: observation +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - dataset + - ../methods_parameters_units/unit + - dimension_type +default_range: string + +classes: + Observation: + is_a: BaseEntity + description: Observation data. + slots: + - dataset_id + - record_type + - record_id + - parameter_id + - value + - unit_id + - dimension_type_id + - dimension_value + - dimension_unit_id + - note + +slots: + dataset_id: + range: Dataset + description: Reference to Dataset. + + record_type: + range: string + description: Type of record. + + record_id: + range: integer + description: ID of the record. + + parameter_id: + range: Parameter + description: Reference to Parameter. + + value: + range: decimal + description: Value of the observation. + + unit_id: + range: Unit + description: Reference to Unit. + + dimension_type_id: + range: DimensionType + description: Reference to DimensionType. + + dimension_value: + range: decimal + description: Value of the dimension. + + dimension_unit_id: + range: Unit + description: Reference to Unit. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/geography.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/geography.yaml deleted file mode 100644 index 0b6ec0eb..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/geography.yaml +++ /dev/null @@ -1,109 +0,0 @@ -id: https://w3id.org/ca_biositing/geography -name: geography -title: Geography Definitions -description: Geospatial entities and locations. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - -classes: - Geography: - description: Geographic region definition (e.g. county, state). - slots: - - geoid - - state_name - - state_fips - - county_name - - county_fips - - region_name - - agg_level_desc - slot_usage: - geoid: - identifier: true - - LocationAddress: - is_a: BaseEntity - description: Specific physical location. - slots: - - geography_id - - address_line1 - - address_line2 - - city - - zip - - lat - - lon - - is_anonymous - -slots: - geoid: - range: string - description: Geographic identifier. - - state_name: - range: string - description: Name of the state. - - state_fips: - range: string - description: FIPS code of the state. - - county_name: - range: string - description: Name of the county. - - county_fips: - range: string - description: FIPS code of the county. - - region_name: - range: string - description: Name of the region. - - agg_level_desc: - range: string - description: Description of the aggregation level. - - geography_id: - range: string - description: Reference to Geography geoid. - - address_line1: - range: string - description: Address line 1. - - address_line2: - range: string - description: Address line 2. - - city: - range: string - description: City name. - - zip: - range: string - description: ZIP code. - - lat: - range: float - description: Latitude. - - lon: - range: float - description: Longitude. - - is_anonymous: - range: boolean - description: Whether the location is anonymous. - - geom: - range: string # TODO: Define geometry type properly, maybe using a custom type or string for WKT - description: Geometry of the polygon. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure.yaml deleted file mode 100644 index 6ce9edf9..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure.yaml +++ /dev/null @@ -1,1069 +0,0 @@ -id: https://w3id.org/ca_biositing/infrastructure -name: infrastructure -title: Infrastructure -description: Infrastructure and facility data. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - general_analysis - - places - - resource_information - -classes: - FacilityRecord: - is_a: BaseEntity - description: Facility record. - slots: - - dataset_id - - facility_name - - location_id - - capacity_mw - - resource_id - - operator - - start_year - - note - - InfrastructureBiodieselPlants: - description: Biodiesel plants infrastructure. - slots: - - biodiesel_plant_id - - company - - bbi_index - - city - - state - - capacity_mmg_per_y - - feedstock - - status - - address - - coordinates - - latitude - - longitude - - source - slot_usage: - biodiesel_plant_id: - identifier: true - range: integer - - InfrastructureBiosolidsFacilities: - description: Biosolids facilities infrastructure. - slots: - - biosolid_facility_id - - report_submitted_date - - latitude - - longitude - - facility - - authority - - plant_type - - aqmd - - facility_address - - facility_city - - state - - facility_zip - - facility_county - - mailing_street_1 - - mailing_city - - mailing_state - - mailing_zip - - biosolids_number - - biosolids_contact - - biosolids_contact_phone - - biosolids_contact_email - - adwf - - potw_biosolids_generated - - twtds_biosolids_treated - - class_b_land_app - - class_b_applier - - class_a_compost - - class_a_heat_dried - - class_a_other - - class_a_other_applier - - twtds_transfer_to_second_preparer - - twtds_second_preparer_name - - adc_or_final_c - - landfill - - landfill_name - - surface_disposal - - deepwell_injection - - stored - - longterm_treatment - - other - - name_of_other - - incineration - slot_usage: - biosolid_facility_id: - identifier: true - range: integer - - InfrastructureCafoManureLocations: - description: CAFO manure locations infrastructure. - slots: - - cafo_manure_id - - latitude - - longitude - - owner_name - - facility_name - - address - - town - - state - - zip - - animal - - animal_feed_operation_type - - animal_units - - animal_count - - manure_total_solids - - source - - date_accessed - slot_usage: - cafo_manure_id: - identifier: true - range: integer - - InfrastructureEthanolBiorefineries: - description: Ethanol biorefineries infrastructure. - slots: - - ethanol_biorefinery_id - - name - - city - - state - - address - - type - - capacity_mgy - - production_mgy - - constr_exp - slot_usage: - ethanol_biorefinery_id: - identifier: true - range: integer - - InfrastructureLandfills: - description: Landfills infrastructure. - slots: - - project_id - - project_int_id - - ghgrp_id - - landfill_id - - landfill_name - - state - - physical_address - - city - - county - - zip_code - - latitude - - longitude - - ownership_type - - landfill_owner_orgs - - landfill_opened_year - - landfill_closure_year - - landfill_status - - waste_in_place - - waste_in_place_year - - lfg_system_in_place - - lfg_collected - - lfg_flared - - project_status - - project_name - - project_start_date - - project_shutdown_date - - project_type_category - - lfg_energy_project_type - - rng_delivery_method - - actual_mw_generation - - rated_mw_capacity - - lfg_flow_to_project - - direct_emission_reductions - - avoided_emission_reductions - slot_usage: - project_id: - identifier: true - range: string - - InfrastructureLivestockAnaerobicDigesters: - description: Livestock anaerobic digesters infrastructure. - slots: - - digester_id - - project_name - - project_type - - city - - state - - digester_type - - profile - - year_operational - - animal_type_class - - animal_types - - pop_feeding_digester - - total_pop_feeding_digester - - cattle - - dairy - - poultry - - swine - - codigestion - - biogas_generation_estimate - - electricity_generated - - biogas_end_uses - - methane_emission_reductions - - latitude - - longitude - slot_usage: - digester_id: - identifier: true - range: integer - - InfrastructureSafAndRenewableDieselPlants: - description: SAF and renewable diesel plants infrastructure. - slots: - - ibcc_index - - company - - city - - state - - country - - capacity - - feedstock - - products - - status - - address - - coordinates - - latitude - - longitude - slot_usage: - ibcc_index: - identifier: true - range: integer - - InfrastructureWastewaterTreatmentPlants: - description: Wastewater treatment plants infrastructure. - slots: - - plant_id - - name - - state - - codigestion - - flow_design_adjusted - - flow_average - - biosolids - - excess_flow - - biogas_utilized - - flaring - - excess_mass_loading_rate - - excess_mass_loading_rate_wet - - methane_production - - energy_content - - electric_kw - - thermal_mmbtu_d - - electric_kwh - - thermal_annual_mmbtu_y - - anaerobic_digestion_facility - - county - - dayload_bdt - - dayload - - equivalent_generation - - facility_type - - feedstock - - type - - city - - latitude - - longitude - - zipcode - slot_usage: - plant_id: - identifier: true - range: integer - - InfrastructureCombustionPlants: - description: Combustion plants infrastructure. - slots: - - combustion_fid - - objectid - - status - - city - - name - - county - - equivalent_generation - - np_mw - - cf - - yearload - - fuel - - notes - - type - - wkt_geom - - geom - - latitude - - longitude - slot_usage: - combustion_fid: - identifier: true - range: integer - - InfrastructureDistrictEnergySystems: - description: District energy systems infrastructure. - slots: - - des_fid - - cbg_id - - name - - system - - object_id - - city - - state - - primary_fuel - - secondary_fuel - - usetype - - cap_st - - cap_hw - - cap_cw - - chpcg_cap - - excess_c - - excess_h - - type - - wkt_geom - - geom - - latitude - - longitude - slot_usage: - des_fid: - identifier: true - range: integer - - InfrastructureFoodProcessingFacilities: - description: Food processing facilities infrastructure. - slots: - - processing_facility_id - - address - - county - - city - - company - - join_count - - master_type - - state - - subtype - - target_fid - - processing_type - - zip - - type - - wkt_geom - - geom - - latitude - - longitude - slot_usage: - processing_facility_id: - identifier: true - range: integer - - InfrastructureMswToEnergyAnaerobicDigesters: - description: MSW to energy anaerobic digesters infrastructure. - slots: - - wte_id - - city - - county - - equivalent_generation - - feedstock - - dayload - - dayload_bdt - - facility_type - - status - - notes - - source - - type - - wkt_geom - - geom - - latitude - - longitude - slot_usage: - wte_id: - identifier: true - range: integer - -slots: - facility_name: - range: string - description: Name of the facility. - - capacity_mw: - range: decimal - description: Capacity in MW. - - operator: - range: string - description: Operator of the facility. - - start_year: - range: integer - description: Start year of the facility. - - biodiesel_plant_id: - range: integer - description: ID of the biodiesel plant. - - company: - range: string - description: Company name. - - bbi_index: - range: integer - description: BBI index. - - city: - range: string - description: City. - - state: - range: string - description: State. - - capacity_mmg_per_y: - range: integer - description: Capacity in MMG per year. - - feedstock: - range: string - description: Feedstock. - - status: - range: string - description: Status. - - address: - range: string - description: Address. - - coordinates: - range: string - description: Coordinates. - - latitude: - range: decimal - description: Latitude. - - longitude: - range: decimal - description: Longitude. - - source: - range: string - description: Source. - - biosolid_facility_id: - range: integer - description: ID of the biosolids facility. - - report_submitted_date: - range: date - description: Date report submitted. - - facility: - range: string - description: Facility name. - - authority: - range: string - description: Authority. - - plant_type: - range: string - description: Plant type. - - aqmd: - range: string - description: AQMD. - - facility_address: - range: string - description: Facility address. - - facility_city: - range: string - description: Facility city. - - facility_zip: - range: string - description: Facility zip. - - facility_county: - range: string - description: Facility county. - - mailing_street_1: - range: string - description: Mailing street 1. - - mailing_city: - range: string - description: Mailing city. - - mailing_state: - range: string - description: Mailing state. - - mailing_zip: - range: string - description: Mailing zip. - - biosolids_number: - range: string - description: Biosolids number. - - biosolids_contact: - range: string - description: Biosolids contact. - - biosolids_contact_phone: - range: string - description: Biosolids contact phone. - - biosolids_contact_email: - range: string - description: Biosolids contact email. - - adwf: - range: decimal - description: ADWF. - - potw_biosolids_generated: - range: integer - description: POTW biosolids generated. - - twtds_biosolids_treated: - range: integer - description: TWTDS biosolids treated. - - class_b_land_app: - range: integer - description: Class B land application. - - class_b_applier: - range: string - description: Class B applier. - - class_a_compost: - range: integer - description: Class A compost. - - class_a_heat_dried: - range: integer - description: Class A heat dried. - - class_a_other: - range: integer - description: Class A other. - - class_a_other_applier: - range: string - description: Class A other applier. - - twtds_transfer_to_second_preparer: - range: integer - description: TWTDS transfer to second preparer. - - twtds_second_preparer_name: - range: string - description: TWTDS second preparer name. - - adc_or_final_c: - range: integer - description: ADC or final cover. - - landfill: - range: integer - description: Landfill. - - landfill_name: - range: string - description: Landfill name. - - surface_disposal: - range: integer - description: Surface disposal. - - deepwell_injection: - range: string - description: Deepwell injection. - - stored: - range: integer - description: Stored. - - longterm_treatment: - range: integer - description: Longterm treatment. - - other: - range: integer - description: Other. - - name_of_other: - range: string - description: Name of other. - - incineration: - range: integer - description: Incineration. - - cafo_manure_id: - range: integer - description: ID of the CAFO manure location. - - owner_name: - range: string - description: Owner name. - - town: - range: string - description: Town. - - zip: - range: string - description: Zip code. - - animal: - range: string - description: Animal. - - animal_feed_operation_type: - range: string - description: Animal feed operation type. - - animal_units: - range: integer - description: Animal units. - - animal_count: - range: integer - description: Animal count. - - manure_total_solids: - range: decimal - description: Manure total solids. - - date_accessed: - range: date - description: Date accessed. - - ethanol_biorefinery_id: - range: integer - description: ID of the ethanol biorefinery. - - type: - range: string - description: Type. - - capacity_mgy: - range: integer - description: Capacity in MGY. - - production_mgy: - range: integer - description: Production in MGY. - - constr_exp: - range: integer - description: Construction/expansion. - - project_id: - range: string - description: Project ID. - - project_int_id: - range: integer - description: Project integer ID. - - ghgrp_id: - range: string - description: GHGRP ID. - - landfill_id: - range: integer - description: Landfill ID. - - physical_address: - range: string - description: Physical address. - - county: - range: string - description: County. - - zip_code: - range: string - description: Zip code. - - ownership_type: - range: string - description: Ownership type. - - landfill_owner_orgs: - range: string - description: Landfill owner organizations. - - landfill_opened_year: - range: date - description: Landfill opened year. - - landfill_closure_year: - range: date - description: Landfill closure year. - - landfill_status: - range: string - description: Landfill status. - - waste_in_place: - range: integer - description: Waste in place. - - waste_in_place_year: - range: date - description: Waste in place year. - - lfg_system_in_place: - range: boolean - description: LFG system in place. - - lfg_collected: - range: decimal - description: LFG collected. - - lfg_flared: - range: decimal - description: LFG flared. - - project_status: - range: string - description: Project status. - - project_name: - range: string - description: Project name. - - project_start_date: - range: date - description: Project start date. - - project_shutdown_date: - range: date - description: Project shutdown date. - - project_type_category: - range: string - description: Project type category. - - lfg_energy_project_type: - range: string - description: LFG energy project type. - - rng_delivery_method: - range: string - description: RNG delivery method. - - actual_mw_generation: - range: decimal - description: Actual MW generation. - - rated_mw_capacity: - range: decimal - description: Rated MW capacity. - - lfg_flow_to_project: - range: decimal - description: LFG flow to project. - - direct_emission_reductions: - range: decimal - description: Direct emission reductions. - - avoided_emission_reductions: - range: decimal - description: Avoided emission reductions. - - digester_id: - range: integer - description: ID of the digester. - - project_type: - range: string - description: Project type. - - digester_type: - range: string - description: Digester type. - - profile: - range: string - description: Profile. - - year_operational: - range: date - description: Year operational. - - animal_type_class: - range: string - description: Animal type class. - - animal_types: - range: string - description: Animal types. - - pop_feeding_digester: - range: string - description: Population feeding digester. - - total_pop_feeding_digester: - range: integer - description: Total population feeding digester. - - cattle: - range: integer - description: Cattle. - - dairy: - range: integer - description: Dairy. - - poultry: - range: integer - description: Poultry. - - swine: - range: integer - description: Swine. - - codigestion: - range: string - description: Codigestion. - - biogas_generation_estimate: - range: integer - description: Biogas generation estimate. - - electricity_generated: - range: integer - description: Electricity generated. - - biogas_end_uses: - range: string - description: Biogas end uses. - - methane_emission_reductions: - range: integer - description: Methane emission reductions. - - ibcc_index: - range: integer - description: IBCC index. - - country: - range: string - description: Country. - - capacity: - range: string - description: Capacity. - - products: - range: string - description: Products. - - plant_id: - range: integer - description: ID of the plant. - - flow_design_adjusted: - range: decimal - description: Flow design adjusted. - - flow_average: - range: decimal - description: Flow average. - - biosolids: - range: decimal - description: Biosolids. - - excess_flow: - range: decimal - description: Excess flow. - - biogas_utilized: - range: boolean - description: Biogas utilized. - - flaring: - range: boolean - description: Flaring. - - excess_mass_loading_rate: - range: decimal - description: Excess mass loading rate. - - excess_mass_loading_rate_wet: - range: decimal - description: Excess mass loading rate wet. - - methane_production: - range: decimal - description: Methane production. - - energy_content: - range: decimal - description: Energy content. - - electric_kw: - range: decimal - description: Electric KW. - - thermal_mmbtu_d: - range: decimal - description: Thermal MMBTU/d. - - electric_kwh: - range: decimal - description: Electric KWh. - - thermal_annual_mmbtu_y: - range: decimal - description: Thermal annual MMBTU/y. - - anaerobic_digestion_facility: - range: string - description: Anaerobic digestion facility. - - dayload_bdt: - range: decimal - description: Dayload BDT. - - dayload: - range: decimal - description: Dayload. - - equivalent_generation: - range: decimal - description: Equivalent generation. - - facility_type: - range: string - description: Facility type. - - zipcode: - range: string - description: Zipcode. - - combustion_fid: - range: integer - description: Combustion FID. - - objectid: - range: integer - description: Object ID. - - np_mw: - range: decimal - description: NP MW. - - cf: - range: decimal - description: CF. - - yearload: - range: integer - description: Yearload. - - fuel: - range: string - description: Fuel. - - notes: - range: string - description: Notes. - - wkt_geom: - range: string - description: WKT geometry. - - geom: - range: string - description: Geometry. - - des_fid: - range: integer - description: DES FID. - - cbg_id: - range: integer - description: CBG ID. - - system: - range: string - description: System. - - object_id: - range: integer - description: Object ID. - - primary_fuel: - range: string - description: Primary fuel. - - secondary_fuel: - range: string - description: Secondary fuel. - - usetype: - range: string - description: Use type. - - cap_st: - range: decimal - description: Cap ST. - - cap_hw: - range: decimal - description: Cap HW. - - cap_cw: - range: decimal - description: Cap CW. - - chpcg_cap: - range: decimal - description: CHPCG Cap. - - excess_c: - range: decimal - description: Excess C. - - excess_h: - range: decimal - description: Excess H. - - processing_facility_id: - range: integer - description: Processing facility ID. - - join_count: - range: integer - description: Join count. - - master_type: - range: string - description: Master type. - - subtype: - range: string - description: Subtype. - - target_fid: - range: integer - description: Target FID. - - processing_type: - range: string - description: Processing type. - - wte_id: - range: integer - description: WTE ID. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/facility_record.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/facility_record.yaml new file mode 100644 index 00000000..f5aebba0 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/facility_record.yaml @@ -0,0 +1,54 @@ +id: https://w3id.org/ca_biositing/infrastructure/facility_record +name: facility_record +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../general_analysis/dataset + - ../places/location_address + - ../resource_information/resource +default_range: string + +classes: + FacilityRecord: + is_a: BaseEntity + description: Facility record. + slots: + - dataset_id + - facility_name + - location_id + - capacity_mw + - resource_id + - operator + - start_year + - note + +slots: + dataset_id: + range: Dataset + description: Reference to Dataset. + + facility_name: + range: string + description: Name of the facility. + + location_id: + range: LocationAddress + description: Reference to LocationAddress. + + capacity_mw: + range: decimal + description: Capacity in MW. + + resource_id: + range: Resource + description: Reference to Resource. + + operator: + range: string + description: Operator of the facility. + + start_year: + range: integer + description: Start year of the facility. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biodiesel_plants.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biodiesel_plants.yaml new file mode 100644 index 00000000..12e5137d --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biodiesel_plants.yaml @@ -0,0 +1,83 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_biodiesel_plants +name: infrastructure_biodiesel_plants +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureBiodieselPlants: + description: Biodiesel plants infrastructure. + slots: + - biodiesel_plant_id + - company + - bbi_index + - city + - state + - capacity_mmg_per_y + - feedstock + - status + - address + - coordinates + - latitude + - longitude + - source + slot_usage: + biodiesel_plant_id: + identifier: true + range: integer + +slots: + biodiesel_plant_id: + range: integer + description: ID of the biodiesel plant. + + company: + range: string + description: Company name. + + bbi_index: + range: integer + description: BBI index. + + city: + range: string + description: City. + + state: + range: string + description: State. + + capacity_mmg_per_y: + range: integer + description: Capacity in MMG per year. + + feedstock: + range: string + description: Feedstock. + + status: + range: string + description: Status. + + address: + range: string + description: Address. + + coordinates: + range: string + description: Coordinates. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. + + source: + range: string + description: Source. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biosolids_facilities.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biosolids_facilities.yaml new file mode 100644 index 00000000..4c74b6bf --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_biosolids_facilities.yaml @@ -0,0 +1,228 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_biosolids_facilities +name: infrastructure_biosolids_facilities +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureBiosolidsFacilities: + description: Biosolids facilities infrastructure. + slots: + - biosolid_facility_id + - report_submitted_date + - latitude + - longitude + - facility + - authority + - plant_type + - aqmd + - facility_address + - facility_city + - state + - facility_zip + - facility_county + - mailing_street_1 + - mailing_city + - mailing_state + - mailing_zip + - biosolids_number + - biosolids_contact + - biosolids_contact_phone + - biosolids_contact_email + - adwf + - potw_biosolids_generated + - twtds_biosolids_treated + - class_b_land_app + - class_b_applier + - class_a_compost + - class_a_heat_dried + - class_a_other + - class_a_other_applier + - twtds_transfer_to_second_preparer + - twtds_second_preparer_name + - adc_or_final_c + - landfill + - landfill_name + - surface_disposal + - deepwell_injection + - stored + - longterm_treatment + - other + - name_of_other + - incineration + slot_usage: + biosolid_facility_id: + identifier: true + range: integer + +slots: + biosolid_facility_id: + range: integer + description: ID of the biosolids facility. + + report_submitted_date: + range: date + description: Date report submitted. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. + + facility: + range: string + description: Facility name. + + authority: + range: string + description: Authority. + + plant_type: + range: string + description: Plant type. + + aqmd: + range: string + description: AQMD. + + facility_address: + range: string + description: Facility address. + + facility_city: + range: string + description: Facility city. + + state: + range: string + description: State. + + facility_zip: + range: string + description: Facility zip. + + facility_county: + range: string + description: Facility county. + + mailing_street_1: + range: string + description: Mailing street 1. + + mailing_city: + range: string + description: Mailing city. + + mailing_state: + range: string + description: Mailing state. + + mailing_zip: + range: string + description: Mailing zip. + + biosolids_number: + range: string + description: Biosolids number. + + biosolids_contact: + range: string + description: Biosolids contact. + + biosolids_contact_phone: + range: string + description: Biosolids contact phone. + + biosolids_contact_email: + range: string + description: Biosolids contact email. + + adwf: + range: decimal + description: ADWF. + + potw_biosolids_generated: + range: integer + description: POTW biosolids generated. + + twtds_biosolids_treated: + range: integer + description: TWTDS biosolids treated. + + class_b_land_app: + range: integer + description: Class B land application. + + class_b_applier: + range: string + description: Class B applier. + + class_a_compost: + range: integer + description: Class A compost. + + class_a_heat_dried: + range: integer + description: Class A heat dried. + + class_a_other: + range: integer + description: Class A other. + + class_a_other_applier: + range: string + description: Class A other applier. + + twtds_transfer_to_second_preparer: + range: integer + description: TWTDS transfer to second preparer. + + twtds_second_preparer_name: + range: string + description: TWTDS second preparer name. + + adc_or_final_c: + range: integer + description: ADC or final cover. + + landfill: + range: integer + description: Landfill. + + landfill_name: + range: string + description: Landfill name. + + surface_disposal: + range: integer + description: Surface disposal. + + deepwell_injection: + range: string + description: Deepwell injection. + + stored: + range: integer + description: Stored. + + longterm_treatment: + range: integer + description: Longterm treatment. + + other: + range: integer + description: Other. + + name_of_other: + range: string + description: Name of other. + + incineration: + range: integer + description: Incineration. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_cafo_manure_locations.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_cafo_manure_locations.yaml new file mode 100644 index 00000000..41222103 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_cafo_manure_locations.yaml @@ -0,0 +1,98 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_cafo_manure_locations +name: infrastructure_cafo_manure_locations +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureCafoManureLocations: + description: CAFO manure locations infrastructure. + slots: + - cafo_manure_id + - latitude + - longitude + - owner_name + - facility_name + - address + - town + - state + - zip + - animal + - animal_feed_operation_type + - animal_units + - animal_count + - manure_total_solids + - source + - date_accessed + slot_usage: + cafo_manure_id: + identifier: true + range: integer + +slots: + cafo_manure_id: + range: integer + description: ID of the CAFO manure location. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. + + owner_name: + range: string + description: Owner name. + + facility_name: + range: string + description: Facility name. + + address: + range: string + description: Address. + + town: + range: string + description: Town. + + state: + range: string + description: State. + + zip: + range: string + description: Zip code. + + animal: + range: string + description: Animal. + + animal_feed_operation_type: + range: string + description: Animal feed operation type. + + animal_units: + range: integer + description: Animal units. + + animal_count: + range: integer + description: Animal count. + + manure_total_solids: + range: decimal + description: Manure total solids. + + source: + range: string + description: Source. + + date_accessed: + range: date + description: Date accessed. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_combustion_plants.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_combustion_plants.yaml new file mode 100644 index 00000000..82ec57af --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_combustion_plants.yaml @@ -0,0 +1,103 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_combustion_plants +name: infrastructure_combustion_plants +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureCombustionPlants: + description: Combustion plants infrastructure. + slots: + - combustion_fid + - objectid + - status + - city + - name + - county + - equivalent_generation + - np_mw + - cf + - yearload + - fuel + - notes + - type + - wkt_geom + - geom + - latitude + - longitude + slot_usage: + combustion_fid: + identifier: true + range: integer + +slots: + combustion_fid: + range: integer + description: Combustion FID. + + objectid: + range: integer + description: Object ID. + + status: + range: string + description: Status. + + city: + range: string + description: City. + + name: + range: string + description: Name. + + county: + range: string + description: County. + + equivalent_generation: + range: decimal + description: Equivalent generation. + + np_mw: + range: decimal + description: NP MW. + + cf: + range: decimal + description: CF. + + yearload: + range: integer + description: Yearload. + + fuel: + range: string + description: Fuel. + + notes: + range: string + description: Notes. + + type: + range: string + description: Type. + + wkt_geom: + range: string + description: WKT geometry. + + geom: + range: string + description: Geometry. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_district_energy_systems.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_district_energy_systems.yaml new file mode 100644 index 00000000..bab9f9f3 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_district_energy_systems.yaml @@ -0,0 +1,123 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_district_energy_systems +name: infrastructure_district_energy_systems +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureDistrictEnergySystems: + description: District energy systems infrastructure. + slots: + - des_fid + - cbg_id + - name + - system + - object_id + - city + - state + - primary_fuel + - secondary_fuel + - usetype + - cap_st + - cap_hw + - cap_cw + - chpcg_cap + - excess_c + - excess_h + - type + - wkt_geom + - geom + - latitude + - longitude + slot_usage: + des_fid: + identifier: true + range: integer + +slots: + des_fid: + range: integer + description: DES FID. + + cbg_id: + range: integer + description: CBG ID. + + name: + range: string + description: Name. + + system: + range: string + description: System. + + object_id: + range: integer + description: Object ID. + + city: + range: string + description: City. + + state: + range: string + description: State. + + primary_fuel: + range: string + description: Primary fuel. + + secondary_fuel: + range: string + description: Secondary fuel. + + usetype: + range: string + description: Use type. + + cap_st: + range: decimal + description: Cap ST. + + cap_hw: + range: decimal + description: Cap HW. + + cap_cw: + range: decimal + description: Cap CW. + + chpcg_cap: + range: decimal + description: CHPCG Cap. + + excess_c: + range: decimal + description: Excess C. + + excess_h: + range: decimal + description: Excess H. + + type: + range: string + description: Type. + + wkt_geom: + range: string + description: WKT geometry. + + geom: + range: string + description: Geometry. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_ethanol_biorefineries.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_ethanol_biorefineries.yaml new file mode 100644 index 00000000..eb68fc54 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_ethanol_biorefineries.yaml @@ -0,0 +1,63 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_ethanol_biorefineries +name: infrastructure_ethanol_biorefineries +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureEthanolBiorefineries: + description: Ethanol biorefineries infrastructure. + slots: + - ethanol_biorefinery_id + - name + - city + - state + - address + - type + - capacity_mgy + - production_mgy + - constr_exp + slot_usage: + ethanol_biorefinery_id: + identifier: true + range: integer + +slots: + ethanol_biorefinery_id: + range: integer + description: ID of the ethanol biorefinery. + + name: + range: string + description: Name. + + city: + range: string + description: City. + + state: + range: string + description: State. + + address: + range: string + description: Address. + + type: + range: string + description: Type. + + capacity_mgy: + range: integer + description: Capacity in MGY. + + production_mgy: + range: integer + description: Production in MGY. + + constr_exp: + range: integer + description: Construction/expansion. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_food_processing_facilities.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_food_processing_facilities.yaml new file mode 100644 index 00000000..30835964 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_food_processing_facilities.yaml @@ -0,0 +1,103 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_food_processing_facilities +name: infrastructure_food_processing_facilities +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureFoodProcessingFacilities: + description: Food processing facilities infrastructure. + slots: + - processing_facility_id + - address + - county + - city + - company + - join_count + - master_type + - state + - subtype + - target_fid + - processing_type + - zip + - type + - wkt_geom + - geom + - latitude + - longitude + slot_usage: + processing_facility_id: + identifier: true + range: integer + +slots: + processing_facility_id: + range: integer + description: Processing facility ID. + + address: + range: string + description: Address. + + county: + range: string + description: County. + + city: + range: string + description: City. + + company: + range: string + description: Company. + + join_count: + range: integer + description: Join count. + + master_type: + range: string + description: Master type. + + state: + range: string + description: State. + + subtype: + range: string + description: Subtype. + + target_fid: + range: integer + description: Target FID. + + processing_type: + range: string + description: Processing type. + + zip: + range: string + description: Zip. + + type: + range: string + description: Type. + + wkt_geom: + range: string + description: WKT geometry. + + geom: + range: string + description: Geometry. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_landfills.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_landfills.yaml new file mode 100644 index 00000000..cee6d298 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_landfills.yaml @@ -0,0 +1,188 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_landfills +name: infrastructure_landfills +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureLandfills: + description: Landfills infrastructure. + slots: + - project_id + - project_int_id + - ghgrp_id + - landfill_id + - landfill_name + - state + - physical_address + - city + - county + - zip_code + - latitude + - longitude + - ownership_type + - landfill_owner_orgs + - landfill_opened_year + - landfill_closure_year + - landfill_status + - waste_in_place + - waste_in_place_year + - lfg_system_in_place + - lfg_collected + - lfg_flared + - project_status + - project_name + - project_start_date + - project_shutdown_date + - project_type_category + - lfg_energy_project_type + - rng_delivery_method + - actual_mw_generation + - rated_mw_capacity + - lfg_flow_to_project + - direct_emission_reductions + - avoided_emission_reductions + slot_usage: + project_id: + identifier: true + range: string + +slots: + project_id: + range: string + description: Project ID. + + project_int_id: + range: integer + description: Project integer ID. + + ghgrp_id: + range: string + description: GHGRP ID. + + landfill_id: + range: integer + description: Landfill ID. + + landfill_name: + range: string + description: Landfill name. + + state: + range: string + description: State. + + physical_address: + range: string + description: Physical address. + + city: + range: string + description: City. + + county: + range: string + description: County. + + zip_code: + range: string + description: Zip code. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. + + ownership_type: + range: string + description: Ownership type. + + landfill_owner_orgs: + range: string + description: Landfill owner organizations. + + landfill_opened_year: + range: date + description: Landfill opened year. + + landfill_closure_year: + range: date + description: Landfill closure year. + + landfill_status: + range: string + description: Landfill status. + + waste_in_place: + range: integer + description: Waste in place. + + waste_in_place_year: + range: date + description: Waste in place year. + + lfg_system_in_place: + range: boolean + description: LFG system in place. + + lfg_collected: + range: decimal + description: LFG collected. + + lfg_flared: + range: decimal + description: LFG flared. + + project_status: + range: string + description: Project status. + + project_name: + range: string + description: Project name. + + project_start_date: + range: date + description: Project start date. + + project_shutdown_date: + range: date + description: Project shutdown date. + + project_type_category: + range: string + description: Project type category. + + lfg_energy_project_type: + range: string + description: LFG energy project type. + + rng_delivery_method: + range: string + description: RNG delivery method. + + actual_mw_generation: + range: decimal + description: Actual MW generation. + + rated_mw_capacity: + range: decimal + description: Rated MW capacity. + + lfg_flow_to_project: + range: decimal + description: LFG flow to project. + + direct_emission_reductions: + range: decimal + description: Direct emission reductions. + + avoided_emission_reductions: + range: decimal + description: Avoided emission reductions. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_livestock_anaerobic_digesters.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_livestock_anaerobic_digesters.yaml new file mode 100644 index 00000000..d8781afa --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_livestock_anaerobic_digesters.yaml @@ -0,0 +1,133 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_livestock_anaerobic_digesters +name: infrastructure_livestock_anaerobic_digesters +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureLivestockAnaerobicDigesters: + description: Livestock anaerobic digesters infrastructure. + slots: + - digester_id + - project_name + - project_type + - city + - state + - digester_type + - profile + - year_operational + - animal_type_class + - animal_types + - pop_feeding_digester + - total_pop_feeding_digester + - cattle + - dairy + - poultry + - swine + - codigestion + - biogas_generation_estimate + - electricity_generated + - biogas_end_uses + - methane_emission_reductions + - latitude + - longitude + slot_usage: + digester_id: + identifier: true + range: integer + +slots: + digester_id: + range: integer + description: ID of the digester. + + project_name: + range: string + description: Project name. + + project_type: + range: string + description: Project type. + + city: + range: string + description: City. + + state: + range: string + description: State. + + digester_type: + range: string + description: Digester type. + + profile: + range: string + description: Profile. + + year_operational: + range: date + description: Year operational. + + animal_type_class: + range: string + description: Animal type class. + + animal_types: + range: string + description: Animal types. + + pop_feeding_digester: + range: string + description: Population feeding digester. + + total_pop_feeding_digester: + range: integer + description: Total population feeding digester. + + cattle: + range: integer + description: Cattle. + + dairy: + range: integer + description: Dairy. + + poultry: + range: integer + description: Poultry. + + swine: + range: integer + description: Swine. + + codigestion: + range: string + description: Codigestion. + + biogas_generation_estimate: + range: integer + description: Biogas generation estimate. + + electricity_generated: + range: integer + description: Electricity generated. + + biogas_end_uses: + range: string + description: Biogas end uses. + + methane_emission_reductions: + range: integer + description: Methane emission reductions. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters.yaml new file mode 100644 index 00000000..f8f1cd29 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters.yaml @@ -0,0 +1,98 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_msw_to_energy_anaerobic_digesters +name: infrastructure_msw_to_energy_anaerobic_digesters +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureMswToEnergyAnaerobicDigesters: + description: MSW to energy anaerobic digesters infrastructure. + slots: + - wte_id + - city + - county + - equivalent_generation + - feedstock + - dayload + - dayload_bdt + - facility_type + - status + - notes + - source + - type + - wkt_geom + - geom + - latitude + - longitude + slot_usage: + wte_id: + identifier: true + range: integer + +slots: + wte_id: + range: integer + description: WTE ID. + + city: + range: string + description: City. + + county: + range: string + description: County. + + equivalent_generation: + range: decimal + description: Equivalent generation. + + feedstock: + range: string + description: Feedstock. + + dayload: + range: decimal + description: Dayload. + + dayload_bdt: + range: decimal + description: Dayload BDT. + + facility_type: + range: string + description: Facility type. + + status: + range: string + description: Status. + + notes: + range: string + description: Notes. + + source: + range: string + description: Source. + + type: + range: string + description: Type. + + wkt_geom: + range: string + description: WKT geometry. + + geom: + range: string + description: Geometry. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_saf_and_renewable_diesel_plants.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_saf_and_renewable_diesel_plants.yaml new file mode 100644 index 00000000..3b32e504 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_saf_and_renewable_diesel_plants.yaml @@ -0,0 +1,83 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_saf_and_renewable_diesel_plants +name: infrastructure_saf_and_renewable_diesel_plants +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureSafAndRenewableDieselPlants: + description: SAF and renewable diesel plants infrastructure. + slots: + - ibcc_index + - company + - city + - state + - country + - capacity + - feedstock + - products + - status + - address + - coordinates + - latitude + - longitude + slot_usage: + ibcc_index: + identifier: true + range: integer + +slots: + ibcc_index: + range: integer + description: IBCC index. + + company: + range: string + description: Company name. + + city: + range: string + description: City. + + state: + range: string + description: State. + + country: + range: string + description: Country. + + capacity: + range: string + description: Capacity. + + feedstock: + range: string + description: Feedstock. + + products: + range: string + description: Products. + + status: + range: string + description: Status. + + address: + range: string + description: Address. + + coordinates: + range: string + description: Coordinates. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_wastewater_treatment_plants.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_wastewater_treatment_plants.yaml new file mode 100644 index 00000000..745ac52b --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/infrastructure/infrastructure_wastewater_treatment_plants.yaml @@ -0,0 +1,168 @@ +id: https://w3id.org/ca_biositing/infrastructure/infrastructure_wastewater_treatment_plants +name: infrastructure_wastewater_treatment_plants +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + InfrastructureWastewaterTreatmentPlants: + description: Wastewater treatment plants infrastructure. + slots: + - plant_id + - name + - state + - codigestion + - flow_design_adjusted + - flow_average + - biosolids + - excess_flow + - biogas_utilized + - flaring + - excess_mass_loading_rate + - excess_mass_loading_rate_wet + - methane_production + - energy_content + - electric_kw + - thermal_mmbtu_d + - electric_kwh + - thermal_annual_mmbtu_y + - anaerobic_digestion_facility + - county + - dayload_bdt + - dayload + - equivalent_generation + - facility_type + - feedstock + - type + - city + - latitude + - longitude + - zipcode + slot_usage: + plant_id: + identifier: true + range: integer + +slots: + plant_id: + range: integer + description: ID of the plant. + + name: + range: string + description: Name. + + state: + range: string + description: State. + + codigestion: + range: string + description: Codigestion. + + flow_design_adjusted: + range: decimal + description: Flow design adjusted. + + flow_average: + range: decimal + description: Flow average. + + biosolids: + range: decimal + description: Biosolids. + + excess_flow: + range: decimal + description: Excess flow. + + biogas_utilized: + range: boolean + description: Biogas utilized. + + flaring: + range: boolean + description: Flaring. + + excess_mass_loading_rate: + range: decimal + description: Excess mass loading rate. + + excess_mass_loading_rate_wet: + range: decimal + description: Excess mass loading rate wet. + + methane_production: + range: decimal + description: Methane production. + + energy_content: + range: decimal + description: Energy content. + + electric_kw: + range: decimal + description: Electric KW. + + thermal_mmbtu_d: + range: decimal + description: Thermal MMBTU/d. + + electric_kwh: + range: decimal + description: Electric KWh. + + thermal_annual_mmbtu_y: + range: decimal + description: Thermal annual MMBTU/y. + + anaerobic_digestion_facility: + range: string + description: Anaerobic digestion facility. + + county: + range: string + description: County. + + dayload_bdt: + range: decimal + description: Dayload BDT. + + dayload: + range: decimal + description: Dayload. + + equivalent_generation: + range: decimal + description: Equivalent generation. + + facility_type: + range: string + description: Facility type. + + feedstock: + range: string + description: Feedstock. + + type: + range: string + description: Type. + + city: + range: string + description: City. + + latitude: + range: decimal + description: Latitude. + + longitude: + range: decimal + description: Longitude. + + zipcode: + range: string + description: Zipcode. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage.yaml deleted file mode 100644 index ed0d6c99..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage.yaml +++ /dev/null @@ -1,88 +0,0 @@ -id: https://w3id.org/ca_biositing/lineage -name: lineage -title: Lineage Definitions -description: Data lineage and ETL tracking. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - -classes: - LineageGroup: - description: Grouping for lineage information. - slots: - - id - - etl_run_id - - note - slot_usage: - id: - identifier: true - range: integer - - EntityLineage: - description: Lineage information for a specific entity. - slots: - - id - - lineage_group_id - - source_table - - source_row_id - - note - slot_usage: - id: - identifier: true - range: integer - - EtlRun: - description: Information about an ETL run. - slots: - - id - - started_at - - completed_at - - pipeline_name - - status - - records_ingested - - note - slot_usage: - id: - identifier: true - -slots: - #Slots for EntityLineage class - - source_table: - range: string - description: Source table name. - - source_row_id: - range: string - description: Source row identifier. - - #Slots for EtlRun class - - started_at: - range: datetime - description: Start time of the ETL run. - - completed_at: - range: datetime - description: Completion time of the ETL run. - - pipeline_name: - range: string - description: Name of the pipeline. - - status: - range: string - description: Status of the ETL run. - - records_ingested: - range: integer - description: Number of records ingested. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/entity_lineage.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/entity_lineage.yaml new file mode 100644 index 00000000..70e0b1fc --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/entity_lineage.yaml @@ -0,0 +1,36 @@ +id: https://w3id.org/ca_biositing/lineage/entity_lineage +name: entity_lineage +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - lineage_group +default_range: string + +classes: + EntityLineage: + description: Lineage information for a specific entity. + slots: + - id + - lineage_group_id + - source_table + - source_row_id + - note + slot_usage: + id: + identifier: true + range: integer + +slots: + lineage_group_id: + range: LineageGroup + description: Reference to the lineage group. + + source_table: + range: string + description: Source table name. + + source_row_id: + range: string + description: Source row identifier. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/etl_run.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/etl_run.yaml new file mode 100644 index 00000000..122bd183 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/etl_run.yaml @@ -0,0 +1,44 @@ +id: https://w3id.org/ca_biositing/lineage/etl_run +name: etl_run +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + EtlRun: + description: Information about an ETL run. + slots: + - id + - started_at + - completed_at + - pipeline_name + - status + - records_ingested + - note + slot_usage: + id: + identifier: true + +slots: + started_at: + range: datetime + description: Start time of the ETL run. + + completed_at: + range: datetime + description: Completion time of the ETL run. + + pipeline_name: + range: string + description: Name of the pipeline. + + status: + range: string + description: Status of the ETL run. + + records_ingested: + range: integer + description: Number of records ingested. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/lineage_group.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/lineage_group.yaml new file mode 100644 index 00000000..3a996250 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/lineage/lineage_group.yaml @@ -0,0 +1,25 @@ +id: https://w3id.org/ca_biositing/lineage/lineage_group +name: lineage_group +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + LineageGroup: + description: Grouping for lineage information. + slots: + - id + - etl_run_id + - note + slot_usage: + id: + identifier: true + range: integer + +slots: + etl_run_id: + range: string + description: Identifier of the ETL run. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units.yaml deleted file mode 100644 index 579609c2..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units.yaml +++ /dev/null @@ -1,130 +0,0 @@ -id: https://w3id.org/ca_biositing/methods_parameters_units -name: methods_parameters_units -title: Methods, Parameters, and Units -description: Analytical methods, parameters, and units of measurement. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - data_sources_metadata - -classes: - Unit: - is_a: LookupBase - description: Unit of measurement. - - Method: - is_a: BaseEntity - description: Analytical method. - slots: - - name - - method_abbrev_id - - method_category_id - - method_standard_id - - description - - detection_limits - - source_id - - MethodAbbrev: - is_a: LookupBase - description: Abbreviation for method. - - MethodCategory: - is_a: LookupBase - description: Category of method. - - MethodStandard: - is_a: LookupBase - description: Standard associated with the method. - - Parameter: - is_a: BaseEntity - description: Parameter being measured. - slots: - - name - - standard_unit_id - - calculated - - description - - ParameterCategory: - is_a: LookupBase - description: Category of parameter. - - ParameterCategoryParameter: - description: Link between Parameter and ParameterCategory. - slots: - - id - - parameter_id - - parameter_category_id - slot_usage: - id: - identifier: true - range: integer - - ParameterUnit: - description: Link between Parameter and Unit (alternate units). - slots: - - id - - parameter_id - - alternate_unit_id - slot_usage: - id: - identifier: true - range: integer - -slots: - #Method slots - - method_abbrev_id: - range: integer - description: Reference to MethodAbbrev. - - method_category_id: - range: integer - description: Reference to MethodCategory. - - method_standard_id: - range: integer - description: Reference to MethodStandard. - - detection_limits: - range: string - description: Detection limits of the method. - - source_id: - range: integer - description: Reference to DataSource. - - #Parameter slots - - standard_unit_id: - range: integer - description: Reference to Unit. - - calculated: - range: boolean - description: Whether the parameter is calculated. - - #ParameterCategoryParameter slots - - parameter_id: - range: integer - description: Reference to Parameter. - - parameter_category_id: - range: integer - description: Reference to ParameterCategory. - - #ParameterUnit slots - - alternate_unit_id: - range: integer - description: Reference to Unit. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method.yaml new file mode 100644 index 00000000..613a296a --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method.yaml @@ -0,0 +1,46 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/method +name: method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../data_sources_metadata/data_source + - method_abbrev + - method_category + - method_standard +default_range: string + +classes: + Method: + is_a: BaseEntity + description: method. + slots: + - name + - method_abbrev_id + - method_category_id + - method_standard_id + - description + - detection_limits + - source_id + +slots: + method_abbrev_id: + range: MethodAbbrev + description: Reference to MethodAbbrev. + + method_category_id: + range: MethodCategory + description: Reference to MethodCategory. + + method_standard_id: + range: MethodStandard + description: Reference to MethodStandard. + + detection_limits: + range: string + description: Detection limits of the method. + + source_id: + range: DataSource + description: Reference to DataSource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_abbrev.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_abbrev.yaml new file mode 100644 index 00000000..a719e522 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_abbrev.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/method_abbrev +name: method_abbrev +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + MethodAbbrev: + is_a: LookupBase + description: Abbreviation for method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_category.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_category.yaml new file mode 100644 index 00000000..ffbe07af --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_category.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/method_category +name: method_category +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + MethodCategory: + is_a: LookupBase + description: Category of method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_standard.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_standard.yaml new file mode 100644 index 00000000..40bb316d --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/method_standard.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/method_standard +name: method_standard +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + MethodStandard: + is_a: LookupBase + description: Standard associated with the method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter.yaml new file mode 100644 index 00000000..d633260c --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter.yaml @@ -0,0 +1,28 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/parameter +name: parameter +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - unit +default_range: string + +classes: + Parameter: + is_a: BaseEntity + description: Parameter being measured. + slots: + - name + - standard_unit_id + - calculated + - description + +slots: + standard_unit_id: + range: Unit + description: Reference to Unit. + + calculated: + range: boolean + description: Whether the parameter is calculated. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category.yaml new file mode 100644 index 00000000..1169b661 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/parameter_category +name: parameter_category +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + ParameterCategory: + is_a: LookupBase + description: Category of parameter. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category_parameter.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category_parameter.yaml new file mode 100644 index 00000000..79b08162 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_category_parameter.yaml @@ -0,0 +1,31 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/parameter_category_parameter +name: parameter_category_parameter +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - parameter + - parameter_category +default_range: string + +classes: + ParameterCategoryParameter: + description: Link between Parameter and ParameterCategory. + slots: + - id + - parameter_id + - parameter_category_id + slot_usage: + id: + identifier: true + range: integer + +slots: + parameter_id: + range: Parameter + description: Reference to Parameter. + + parameter_category_id: + range: ParameterCategory + description: Reference to ParameterCategory. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_unit.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_unit.yaml new file mode 100644 index 00000000..0ed87eb3 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/parameter_unit.yaml @@ -0,0 +1,31 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/parameter_unit +name: parameter_unit +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - parameter + - unit +default_range: string + +classes: + ParameterUnit: + description: Link between Parameter and Unit (alternate units). + slots: + - id + - parameter_id + - alternate_unit_id + slot_usage: + id: + identifier: true + range: integer + +slots: + parameter_id: + range: Parameter + description: Reference to Parameter. + + alternate_unit_id: + range: Unit + description: Reference to Unit. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/unit.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/unit.yaml new file mode 100644 index 00000000..1fea7b71 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/methods_parameters_units/unit.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/methods_parameters_units/unit +name: unit +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + Unit: + is_a: LookupBase + description: Unit of measurement. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/contact.yaml similarity index 53% rename from src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people.yaml rename to src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/contact.yaml index 5f5c59f5..ec33e8de 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/contact.yaml @@ -1,19 +1,11 @@ -id: https://w3id.org/ca_biositing/people -name: people -title: People Definitions -description: Contact and provider information. -license: https://creativecommons.org/publicdomain/zero/1.0/ - +id: https://w3id.org/ca_biositing/people/contact +name: contact prefixes: linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - imports: - linkml:types - - core + - ../core/base_entity +default_range: string classes: Contact: @@ -29,16 +21,6 @@ classes: identifier: true range: integer - Provider: - is_a: BaseEntity - description: Provider information. - slots: - - codename - slot_usage: - id: - identifier: true - range: integer - slots: first_name: range: string @@ -55,7 +37,3 @@ slots: affiliation: range: string description: Affiliation. - - codename: - range: string - description: Codename for the provider. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/provider.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/provider.yaml new file mode 100644 index 00000000..31a9c842 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/people/provider.yaml @@ -0,0 +1,24 @@ +id: https://w3id.org/ca_biositing/people/provider +name: provider +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + Provider: + is_a: BaseEntity + description: Provider information. + slots: + - codename + slot_usage: + id: + identifier: true + range: integer + +slots: + codename: + range: string + description: Codename for the provider. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places.yaml deleted file mode 100644 index 5a6342d3..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places.yaml +++ /dev/null @@ -1,105 +0,0 @@ -id: https://w3id.org/ca_biositing/places -name: places -title: Places Definitions -description: Geographic locations and addresses. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - -classes: - Geography: - description: Geographic location. - slots: - - geoid - - state_name - - state_fips - - county_name - - county_fips - - region_name - - agg_level_desc - slot_usage: - geoid: - identifier: true - - LocationAddress: - is_a: BaseEntity - description: Physical address. - slots: - - geography_id - - address_line1 - - address_line2 - - city - - zip - - lat - - lon - - is_anonymous - -slots: - geoid: - range: string - description: Unique identifier for geography. - - state_name: - range: string - description: State name. - - state_fips: - range: string - description: State FIPS code. - - county_name: - range: string - description: County name. - - county_fips: - range: string - description: County FIPS code. - - region_name: - range: string - description: Region name. - - agg_level_desc: - range: string - description: Aggregation level description. - - geography_id: - range: string - description: Reference to Geography. - - address_line1: - range: string - description: Address line 1. - - address_line2: - range: string - description: Address line 2. - - city: - range: string - description: City. - - zip: - range: string - description: ZIP code. - - lat: - range: float - description: Latitude. - - lon: - range: float - description: Longitude. - - is_anonymous: - range: boolean - description: Whether the location is anonymous. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/geography.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/geography.yaml new file mode 100644 index 00000000..98cc3fcc --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/geography.yaml @@ -0,0 +1,52 @@ +id: https://w3id.org/ca_biositing/places/geography +name: geography +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity +default_range: string + +classes: + Geography: + description: Geographic location. + slots: + - geoid + - state_name + - state_fips + - county_name + - county_fips + - region_name + - agg_level_desc + slot_usage: + geoid: + identifier: true + +slots: + geoid: + range: string + description: Unique identifier for geography. + + state_name: + range: string + description: State name. + + state_fips: + range: string + description: State FIPS code. + + county_name: + range: string + description: County name. + + county_fips: + range: string + description: County FIPS code. + + region_name: + range: string + description: Region name. + + agg_level_desc: + range: string + description: Aggregation level description. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/location_address.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/location_address.yaml new file mode 100644 index 00000000..b967e799 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/places/location_address.yaml @@ -0,0 +1,56 @@ +id: https://w3id.org/ca_biositing/places/location_address +name: location_address +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - geography +default_range: string + +classes: + LocationAddress: + is_a: BaseEntity + description: Physical address. + slots: + - geography_id + - address_line1 + - address_line2 + - city + - zip + - lat + - lon + - is_anonymous + +slots: + geography_id: + range: Geography + description: Reference to Geography. + + address_line1: + range: string + description: Address line 1. + + address_line2: + range: string + description: Address line 2. + + city: + range: string + description: City. + + zip: + range: string + description: ZIP code. + + lat: + range: float + description: Latitude. + + lon: + range: float + description: Longitude. + + is_anonymous: + range: boolean + description: Whether the location is anonymous. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml deleted file mode 100644 index 2c5c9b4c..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/primary_ag_product.yaml +++ /dev/null @@ -1,23 +0,0 @@ -id: https://w3id.org/ca_biositing/primary_ag_product -name: primary_ag_product -title: Primary Agricultural Product -description: Core definitions for primary agricultural products. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - -classes: - PrimaryAgProduct: - is_a: LookupBase - description: Primary agricultural product definition. - slots: - - note diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml deleted file mode 100644 index 9bf95246..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_availability.yaml +++ /dev/null @@ -1,35 +0,0 @@ -id: https://w3id.org/ca_biositing/resource_availability -name: resource_availability -title: Resource Availability -description: Availability of a resource in a location. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - places - - resource_information - -classes: - ResourceAvailability: - is_a: BaseEntity - description: Availability of a resource in a location. - slots: - - resource_id - - geoid - - from_month - - to_month - - year_round - - note - -slots: - resource_id: - range: Resource - description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml deleted file mode 100644 index d346b197..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_counterfactual.yaml +++ /dev/null @@ -1,47 +0,0 @@ -id: https://w3id.org/ca_biositing/resource_counterfactual -name: resource_counterfactual -title: Resource Counterfactual -description: Counterfactual uses of a resource. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - places - - resource_information - - data_sources_metadata - -classes: - ResourceCounterfactual: - is_a: BaseEntity - description: Counterfactual uses of a resource. - slots: - - geoid - - resource_id - - counterfactual_description - - animal_bedding_percent - - animal_bedding_source_id - - animal_feed_percent - - animal_feed_source_id - - bioelectricty_percent - - bioelectricty_source_id - - burn_percent - - burn_source_id - - compost_percent - - compost_source_id - - landfill_percent - - landfill_source_id - - counterfactual_date - - note - -slots: - resource_id: - range: Resource - description: Reference to Resource. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml deleted file mode 100644 index 82237cae..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information.yaml +++ /dev/null @@ -1,189 +0,0 @@ -id: https://w3id.org/ca_biositing/resource_information -name: resource_information -title: Resource Information -description: Biomass resources and their classifications. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - places - -classes: - Resource: - is_a: BaseEntity - description: Biomass resource definition. - slots: - - name - - primary_ag_product_id - - resource_class_id - - resource_subclass_id - - note - - test - - ResourceClass: - is_a: LookupBase - description: Classification of resources. - - ResourceSubclass: - is_a: LookupBase - description: Sub-classification of resources. - - PrimaryAgProduct: - is_a: LookupBase - description: Primary agricultural product definition. - slots: - - note - - ResourceAvailability: - is_a: BaseEntity - description: Availability of a resource in a location. - slots: - - resource_id - - geoid - - from_month - - to_month - - year_round - - note - - ResourceCounterfactual: - is_a: BaseEntity - description: Counterfactual uses of a resource. - slots: - - geoid - - resource_id - - counterfactual_description - - animal_bedding_percent - - animal_bedding_source_id - - animal_feed_percent - - animal_feed_source_id - - bioelectricty_percent - - bioelectricty_source_id - - burn_percent - - burn_source_id - - compost_percent - - compost_source_id - - landfill_percent - - landfill_source_id - - counterfactual_date - - note - - ResourceMorphology: - description: Morphology of a resource. - slots: - - id - - resource_id - - morphology_uri - slot_usage: - id: - identifier: true - range: integer - -slots: - #Resource slots - - primary_ag_product_id: - range: integer - description: Reference to PrimaryAgProduct. - - resource_class_id: - range: integer - description: Reference to ResourceClass. - - resource_subclass_id: - range: integer - description: Reference to ResourceSubclass. - - test: - range: string - description: Test field. - - #ResourceAvailability slots - - resource_id: - range: integer - description: Reference to Resource. - - geoid: - range: string - description: Reference to Geography. - - from_month: - range: integer - description: Start month of availability. - - to_month: - range: integer - description: End month of availability. - - year_round: - range: boolean - description: Whether available year-round. - - #ResourceCounterfactual slots - - counterfactual_description: - range: string - description: Description of counterfactual scenario. - - animal_bedding_percent: - range: decimal - description: Percentage used for animal bedding. - - animal_bedding_source_id: - range: integer - description: Source for animal bedding data. - - animal_feed_percent: - range: decimal - description: Percentage used for animal feed. - - animal_feed_source_id: - range: integer - description: Source for animal feed data. - - bioelectricty_percent: - range: decimal - description: Percentage used for bioelectricity. - - bioelectricty_source_id: - range: integer - description: Source for bioelectricity data. - - burn_percent: - range: decimal - description: Percentage burned. - - burn_source_id: - range: integer - description: Source for burn data. - - compost_percent: - range: decimal - description: Percentage composted. - - compost_source_id: - range: integer - description: Source for compost data. - - landfill_percent: - range: decimal - description: Percentage landfilled. - - landfill_source_id: - range: integer - description: Source for landfill data. - - counterfactual_date: - range: date - description: Date of counterfactual data. - - morphology_uri: - range: string diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/primary_ag_product.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/primary_ag_product.yaml new file mode 100644 index 00000000..94937588 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/primary_ag_product.yaml @@ -0,0 +1,14 @@ +id: https://w3id.org/ca_biositing/resource_information/primary_ag_product +name: primary_ag_product +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + PrimaryAgProduct: + is_a: LookupBase + description: Primary agricultural product definition. + slots: diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource.yaml new file mode 100644 index 00000000..87baf0e7 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource.yaml @@ -0,0 +1,44 @@ +id: https://w3id.org/ca_biositing/resource_information/resource +name: resource +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - primary_ag_product + - resource_class + - resource_subclass +default_range: string + +classes: + Resource: + is_a: BaseEntity + description: Biomass resource definition. + slots: + - name + - primary_ag_product_id + - resource_class_id + - resource_subclass_id + - note + - test + +slots: + primary_ag_product_id: + range: PrimaryAgProduct + description: Reference to PrimaryAgProduct. + + resource_class_id: + range: ResourceClass + description: Reference to ResourceClass. + + resource_subclass_id: + range: ResourceSubclass + description: Reference to ResourceSubclass. + + note: + range: string + description: Additional notes about the resource. + + test: + range: string + description: Test field. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_availability.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_availability.yaml new file mode 100644 index 00000000..761ecccb --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_availability.yaml @@ -0,0 +1,43 @@ +id: https://w3id.org/ca_biositing/resource_information/resource_availability +name: resource_availability +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - resource + - ../places/geography +default_range: string + +classes: + ResourceAvailability: + is_a: BaseEntity + description: Availability of a resource in a location. + slots: + - resource_id + - geoid + - from_month + - to_month + - year_round + - note + +slots: + resource_id: + range: Resource + description: Reference to Resource. + + geoid: + range: Geography + description: Reference to Geography. + + from_month: + range: integer + description: Start month of availability. + + to_month: + range: integer + description: End month of availability. + + year_round: + range: boolean + description: Whether available year-round. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_class.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_class.yaml new file mode 100644 index 00000000..0bace74e --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_class.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/resource_information/resource_class +name: resource_class +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + ResourceClass: + is_a: LookupBase + description: Classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_counterfactual.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_counterfactual.yaml new file mode 100644 index 00000000..62c02afb --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_counterfactual.yaml @@ -0,0 +1,96 @@ +id: https://w3id.org/ca_biositing/resource_information/resource_counterfactual +name: resource_counterfactual +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - resource + - ../places/geography + #- ../data_sources_metadata/data_source +default_range: string + +classes: + ResourceCounterfactual: + description: Counterfactual uses of a resource. + slots: + - geoid + - resource_id + - counterfactual_description + - animal_bedding_percent + - animal_bedding_source_id + - animal_feed_percent + - animal_feed_source_id + - bioelectricty_percent + - bioelectricty_source_id + - burn_percent + - burn_source_id + - compost_percent + - compost_source_id + - landfill_percent + - landfill_source_id + - counterfactual_date + +slots: + geoid: + range: Geography + description: Reference to Geography. + + resource_id: + range: Resource + description: Reference to Resource. + + counterfactual_description: + range: string + description: Description of counterfactual scenario. + + animal_bedding_percent: + range: decimal + description: Percentage used for animal bedding. + + animal_bedding_source_id: + range: integer + description: Source for animal bedding data. + + animal_feed_percent: + range: decimal + description: Percentage used for animal feed. + + animal_feed_source_id: + range: integer + description: Source for animal feed data. + + bioelectricty_percent: + range: decimal + description: Percentage used for bioelectricity. + + bioelectricty_source_id: + range: integer + description: Source for bioelectricity data. + + burn_percent: + range: decimal + description: Percentage burned. + + burn_source_id: + range: integer + description: Source for burn data. + + compost_percent: + range: decimal + description: Percentage composted. + + compost_source_id: + range: integer + description: Source for compost data. + + landfill_percent: + range: decimal + description: Percentage landfilled. + + landfill_source_id: + range: integer + description: Source for landfill data. + + counterfactual_date: + range: date + description: Date of counterfactual data. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_morphology.yaml similarity index 58% rename from src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml rename to src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_morphology.yaml index f9d93c67..11192995 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_morphology.yaml +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_morphology.yaml @@ -1,20 +1,12 @@ -id: https://w3id.org/ca_biositing/resource_morphology +id: https://w3id.org/ca_biositing/resource_information/resource_morphology name: resource_morphology -title: Resource Morphology -description: Morphology of a resource. -license: https://creativecommons.org/publicdomain/zero/1.0/ - prefixes: linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - imports: - linkml:types - - core - - resource_information + - ../core/base_entity + - resource +default_range: string classes: ResourceMorphology: @@ -32,3 +24,6 @@ slots: resource_id: range: Resource description: Reference to Resource. + + morphology_uri: + range: string diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_subclass.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_subclass.yaml new file mode 100644 index 00000000..bbde717b --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_information/resource_subclass.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/resource_information/resource_subclass +name: resource_subclass +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + ResourceSubclass: + is_a: LookupBase + description: Sub-classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml deleted file mode 100644 index 0236bca8..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/resource_subclass.yaml +++ /dev/null @@ -1,21 +0,0 @@ -id: https://w3id.org/ca_biositing/resource_subclass -name: resource_subclass -title: Resource Subclass -description: Sub-classification of resources. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - -classes: - ResourceSubclass: - is_a: LookupBase - description: Sub-classification of resources. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation.yaml deleted file mode 100644 index 758bb6c8..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation.yaml +++ /dev/null @@ -1,70 +0,0 @@ -id: https://w3id.org/ca_biositing/sample_preparation -name: sample_preparation -title: Sample Preparation -description: Sample preparation methods and records. -license: https://creativecommons.org/publicdomain/zero/1.0/ - -prefixes: - linkml: https://w3id.org/linkml/ - ca_biositing: https://w3id.org/ca_biositing/ - -default_prefix: ca_biositing -default_range: string - -imports: - - linkml:types - - core - - field_sampling - - people - -classes: - PreparationMethod: - is_a: BaseEntity - description: Method of sample preparation. - slots: - - name - - description - - prep_method_abbrev_id - - prep_temp_c - - uri - - drying_step - - PreparationMethodAbbreviation: - is_a: LookupBase - description: Abbreviation for preparation method. - - PreparedSample: - is_a: BaseEntity - description: Sample that has been prepared. - slots: - - name - - field_sample_id - - prep_method_id - - prep_date - - preparer_id - - note - -slots: - prep_method_abbrev_id: - range: integer - description: Reference to PreparationMethodAbbreviation. - - prep_temp_c: - range: decimal - description: Preparation temperature in Celsius. - - drying_step: - range: boolean - description: Whether drying step was involved. - - prep_method_id: - range: integer - description: Reference to PreparationMethod. - - prep_date: - range: date - description: Date of preparation. - - preparer_id: - range: integer - description: Reference to Contact (preparer). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method.yaml new file mode 100644 index 00000000..8abf57c2 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method.yaml @@ -0,0 +1,34 @@ +id: https://w3id.org/ca_biositing/sample_preparation/preparation_method +name: preparation_method +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - preparation_method_abbreviation +default_range: string + +classes: + PreparationMethod: + is_a: BaseEntity + description: Method of sample preparation. + slots: + - name + - description + - prep_method_abbrev_id + - prep_temp_c + - uri + - drying_step + +slots: + prep_method_abbrev_id: + range: PreparationMethodAbbreviation + description: Reference to PreparationMethodAbbreviation. + + prep_temp_c: + range: decimal + description: Preparation temperature in Celsius. + + drying_step: + range: boolean + description: Whether drying step was involved. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method_abbreviation.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method_abbreviation.yaml new file mode 100644 index 00000000..9a6c3c01 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/preparation_method_abbreviation.yaml @@ -0,0 +1,13 @@ +id: https://w3id.org/ca_biositing/sample_preparation/preparation_method_abbreviation +name: preparation_method_abbreviation +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/lookup_base +default_range: string + +classes: + PreparationMethodAbbreviation: + is_a: LookupBase + description: Abbreviation for preparation method. diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/prepared_sample.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/prepared_sample.yaml new file mode 100644 index 00000000..6afccf07 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/modules/sample_preparation/prepared_sample.yaml @@ -0,0 +1,40 @@ +id: https://w3id.org/ca_biositing/sample_preparation/prepared_sample +name: prepared_sample +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - ../core/base_entity + - ../field_sampling/field_sample + - preparation_method + - ../people/contact +default_range: string + +classes: + PreparedSample: + is_a: BaseEntity + description: Sample that has been prepared. + slots: + - name + - field_sample_id + - prep_method_id + - prep_date + - preparer_id + - note + +slots: + field_sample_id: + range: FieldSample + description: Reference to FieldSample. + + prep_method_id: + range: PreparationMethod + description: Reference to PreparationMethod. + + prep_date: + range: date + description: Date of preparation. + + preparer_id: + range: Contact + description: Reference to Contact (preparer). diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema1.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema1.yaml new file mode 100644 index 00000000..ad80703a --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema1.yaml @@ -0,0 +1,18 @@ +id: https://example.com/schema1 +name: schema1 +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string + +classes: + ClassA: + slots: + - id + id_prefixes: + - A + +slots: + id: + identifier: true diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema2.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema2.yaml new file mode 100644 index 00000000..0571c541 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema2.yaml @@ -0,0 +1,22 @@ +id: https://example.com/schema2 +name: schema2 +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - schema1 +default_range: string + +classes: + ClassB: + slots: + - id + - class_a_id + id_prefixes: + - B + +slots: + id: + identifier: true + class_a_id: + range: ClassA diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema3.yaml b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema3.yaml new file mode 100644 index 00000000..cbc18b87 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/linkml/test_schemas/schema3.yaml @@ -0,0 +1,26 @@ +id: https://example.com/schema3 +name: schema3 +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types + - schema1 + - schema2 +default_range: string + +classes: + ClassC: + slots: + - id + - class_a_id + - class_b_id + id_prefixes: + - C + +slots: + id: + identifier: true + class_a_id: + range: ClassA + class_b_id: + range: ClassB diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py deleted file mode 100644 index 85b02306..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim1_records.py +++ /dev/null @@ -1,1747 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class ExperimentMethod(Base): - """ - Link between Experiment and Method. - """ - __tablename__ = 'experiment_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - method_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentMethod(id={self.id},experiment_id={self.experiment_id},method_id={self.method_id},)" - - - - - - -class ExperimentEquipment(Base): - """ - Link between Experiment and Equipment. - """ - __tablename__ = 'experiment_equipment' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - equipment_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentEquipment(id={self.id},experiment_id={self.experiment_id},equipment_id={self.equipment_id},)" - - - - - - -class ExperimentAnalysis(Base): - """ - Link between Experiment and AnalysisType. - """ - __tablename__ = 'experiment_analysis' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - analysis_type_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentAnalysis(id={self.id},experiment_id={self.experiment_id},analysis_type_id={self.analysis_type_id},)" - - - - - - -class ExperimentPreparedSample(Base): - """ - Link between Experiment and PreparedSample. - """ - __tablename__ = 'experiment_prepared_sample' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - prepared_sample_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentPreparedSample(id={self.id},experiment_id={self.experiment_id},prepared_sample_id={self.prepared_sample_id},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class Aim1RecordBase(BaseEntity): - """ - - """ - __tablename__ = 'aim1_record_base' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Aim1RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethod(BaseEntity): - """ - Method of sample preparation. - """ - __tablename__ = 'preparation_method' - - name = Column(Text()) - description = Column(Text()) - prep_method_abbrev_id = Column(Integer()) - prep_temp_c = Column(Numeric()) - uri = Column(Text()) - drying_step = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethodAbbreviation(LookupBase): - """ - Abbreviation for preparation method. - """ - __tablename__ = 'preparation_method_abbreviation' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparedSample(BaseEntity): - """ - Sample that has been prepared. - """ - __tablename__ = 'prepared_sample' - - name = Column(Text()) - field_sample_id = Column(Integer()) - prep_method_id = Column(Integer()) - prep_date = Column(Date()) - preparer_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Experiment(BaseEntity): - """ - Experiment definition. - """ - __tablename__ = 'experiment' - - analyst_id = Column(Integer()) - exper_start_date = Column(Date()) - exper_duration = Column(Numeric()) - exper_duration_unit_id = Column(Integer()) - exper_location_id = Column(Integer()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Experiment(analyst_id={self.analyst_id},exper_start_date={self.exper_start_date},exper_duration={self.exper_duration},exper_duration_unit_id={self.exper_duration_unit_id},exper_location_id={self.exper_location_id},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Equipment(LookupBase): - """ - Equipment used in experiments. - """ - __tablename__ = 'equipment' - - equipment_location_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Equipment(equipment_location_id={self.equipment_location_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProximateRecord(Aim1RecordBase): - """ - Proximate analysis record. - """ - __tablename__ = 'proximate_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ProximateRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UltimateRecord(Aim1RecordBase): - """ - Ultimate analysis record. - """ - __tablename__ = 'ultimate_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UltimateRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CompositionalRecord(Aim1RecordBase): - """ - Compositional analysis record. - """ - __tablename__ = 'compositional_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"CompositionalRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class IcpRecord(Aim1RecordBase): - """ - ICP analysis record. - """ - __tablename__ = 'icp_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"IcpRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class XrfRecord(Aim1RecordBase): - """ - XRF analysis record. - """ - __tablename__ = 'xrf_record' - - maybe_wavelength_nm = Column(Numeric()) - maybe_intensity = Column(Numeric()) - maybe_energy_slope = Column(Numeric()) - maybe_energy_offset = Column(Numeric()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"XrfRecord(maybe_wavelength_nm={self.maybe_wavelength_nm},maybe_intensity={self.maybe_intensity},maybe_energy_slope={self.maybe_energy_slope},maybe_energy_offset={self.maybe_energy_offset},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class XrdRecord(Aim1RecordBase): - """ - XRD analysis record. - """ - __tablename__ = 'xrd_record' - - maybe_scan_low_nm = Column(Integer()) - maybe_scan_high_nm = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"XrdRecord(maybe_scan_low_nm={self.maybe_scan_low_nm},maybe_scan_high_nm={self.maybe_scan_high_nm},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CalorimetryRecord(Aim1RecordBase): - """ - Calorimetry analysis record. - """ - __tablename__ = 'calorimetry_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"CalorimetryRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FtnirRecord(Aim1RecordBase): - """ - FT-NIR analysis record. - """ - __tablename__ = 'ftnir_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FtnirRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class RgbRecord(Aim1RecordBase): - """ - RGB analysis record. - """ - __tablename__ = 'rgb_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"RgbRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py deleted file mode 100644 index 374bbf8b..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/aim2_records.py +++ /dev/null @@ -1,1600 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class ExperimentMethod(Base): - """ - Link between Experiment and Method. - """ - __tablename__ = 'experiment_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - method_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentMethod(id={self.id},experiment_id={self.experiment_id},method_id={self.method_id},)" - - - - - - -class ExperimentEquipment(Base): - """ - Link between Experiment and Equipment. - """ - __tablename__ = 'experiment_equipment' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - equipment_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentEquipment(id={self.id},experiment_id={self.experiment_id},equipment_id={self.equipment_id},)" - - - - - - -class ExperimentAnalysis(Base): - """ - Link between Experiment and AnalysisType. - """ - __tablename__ = 'experiment_analysis' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - analysis_type_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentAnalysis(id={self.id},experiment_id={self.experiment_id},analysis_type_id={self.analysis_type_id},)" - - - - - - -class ExperimentPreparedSample(Base): - """ - Link between Experiment and PreparedSample. - """ - __tablename__ = 'experiment_prepared_sample' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - prepared_sample_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentPreparedSample(id={self.id},experiment_id={self.experiment_id},prepared_sample_id={self.prepared_sample_id},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class Aim2RecordBase(BaseEntity): - """ - - """ - __tablename__ = 'aim2_record_base' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Aim2RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Strain(LookupBase): - """ - Strain used in fermentation. - """ - __tablename__ = 'strain' - - parent_strain_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Strain(parent_strain_id={self.parent_strain_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethod(BaseEntity): - """ - Method of sample preparation. - """ - __tablename__ = 'preparation_method' - - name = Column(Text()) - description = Column(Text()) - prep_method_abbrev_id = Column(Integer()) - prep_temp_c = Column(Numeric()) - uri = Column(Text()) - drying_step = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethodAbbreviation(LookupBase): - """ - Abbreviation for preparation method. - """ - __tablename__ = 'preparation_method_abbreviation' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparedSample(BaseEntity): - """ - Sample that has been prepared. - """ - __tablename__ = 'prepared_sample' - - name = Column(Text()) - field_sample_id = Column(Integer()) - prep_method_id = Column(Integer()) - prep_date = Column(Date()) - preparer_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Experiment(BaseEntity): - """ - Experiment definition. - """ - __tablename__ = 'experiment' - - analyst_id = Column(Integer()) - exper_start_date = Column(Date()) - exper_duration = Column(Numeric()) - exper_duration_unit_id = Column(Integer()) - exper_location_id = Column(Integer()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Experiment(analyst_id={self.analyst_id},exper_start_date={self.exper_start_date},exper_duration={self.exper_duration},exper_duration_unit_id={self.exper_duration_unit_id},exper_location_id={self.exper_location_id},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Equipment(LookupBase): - """ - Equipment used in experiments. - """ - __tablename__ = 'equipment' - - equipment_location_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Equipment(equipment_location_id={self.equipment_location_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PretreatmentRecord(Aim2RecordBase): - """ - Pretreatment record. - """ - __tablename__ = 'pretreatment_record' - - pretreatment_method_id = Column(Integer()) - eh_method_id = Column(Integer()) - reaction_block_id = Column(Integer()) - block_position = Column(Text()) - temperature = Column(Numeric()) - replicate_no = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PretreatmentRecord(pretreatment_method_id={self.pretreatment_method_id},eh_method_id={self.eh_method_id},reaction_block_id={self.reaction_block_id},block_position={self.block_position},temperature={self.temperature},replicate_no={self.replicate_no},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FermentationRecord(Aim2RecordBase): - """ - Fermentation record. - """ - __tablename__ = 'fermentation_record' - - strain_id = Column(Integer()) - pretreatment_method_id = Column(Integer()) - eh_method_id = Column(Integer()) - replicate_no = Column(Integer()) - well_position = Column(Text()) - temperature = Column(Numeric()) - agitation_rpm = Column(Numeric()) - vessel_id = Column(Integer()) - analyte_detection_equipment_id = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FermentationRecord(strain_id={self.strain_id},pretreatment_method_id={self.pretreatment_method_id},eh_method_id={self.eh_method_id},replicate_no={self.replicate_no},well_position={self.well_position},temperature={self.temperature},agitation_rpm={self.agitation_rpm},vessel_id={self.vessel_id},analyte_detection_equipment_id={self.analyte_detection_equipment_id},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class GasificationRecord(Aim2RecordBase): - """ - Gasification record. - """ - __tablename__ = 'gasification_record' - - feedstock_mass = Column(Numeric()) - bed_temperature = Column(Numeric()) - gas_flow_rate = Column(Numeric()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"GasificationRecord(feedstock_mass={self.feedstock_mass},bed_temperature={self.bed_temperature},gas_flow_rate={self.gas_flow_rate},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AutoclaveRecord(Aim2RecordBase): - """ - Autoclave record. - """ - __tablename__ = 'autoclave_record' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"AutoclaveRecord(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py index 6fc7680f..0fcd6b01 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py @@ -50,143 +50,38 @@ def __repr__(self): -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class LineageGroup(Base): - """ - Grouping for lineage information. - """ - __tablename__ = 'lineage_group' - - id = Column(Integer(), primary_key=True, nullable=False ) - etl_run_id = Column(Text()) - note = Column(Text()) - - - def __repr__(self): - return f"LineageGroup(id={self.id},etl_run_id={self.etl_run_id},note={self.note},)" - - - - - - -class EntityLineage(Base): - """ - Lineage information for a specific entity. - """ - __tablename__ = 'entity_lineage' - - id = Column(Integer(), primary_key=True, nullable=False ) - lineage_group_id = Column(Integer()) - source_table = Column(Text()) - source_row_id = Column(Text()) - note = Column(Text()) - - - def __repr__(self): - return f"EntityLineage(id={self.id},lineage_group_id={self.lineage_group_id},source_table={self.source_table},source_row_id={self.source_row_id},note={self.note},)" - - - - - - -class EtlRun(Base): - """ - Information about an ETL run. - """ - __tablename__ = 'etl_run' - - id = Column(Integer(), primary_key=True, nullable=False ) - started_at = Column(DateTime()) - completed_at = Column(DateTime()) - pipeline_name = Column(Text()) - status = Column(Text()) - records_ingested = Column(Integer()) - note = Column(Text()) - - - def __repr__(self): - return f"EtlRun(id={self.id},started_at={self.started_at},completed_at={self.completed_at},pipeline_name={self.pipeline_name},status={self.status},records_ingested={self.records_ingested},note={self.note},)" - - - - - - -class ParameterCategoryParameter(Base): +class ExperimentAnalysis(Base): """ - Link between Parameter and ParameterCategory. + Link between Experiment and AnalysisType. """ - __tablename__ = 'parameter_category_parameter' + __tablename__ = 'experiment_analysis' id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + analysis_type_id = Column(Integer(), ForeignKey('AnalysisType.id')) def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" + return f"ExperimentAnalysis(id={self.id},experiment_id={self.experiment_id},analysis_type_id={self.analysis_type_id},)" -class ParameterUnit(Base): +class ExperimentEquipment(Base): """ - Link between Parameter and Unit (alternate units). + Link between Experiment and Equipment. """ - __tablename__ = 'parameter_unit' + __tablename__ = 'experiment_equipment' id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + equipment_id = Column(Integer(), ForeignKey('Equipment.id')) def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" + return f"ExperimentEquipment(id={self.id},experiment_id={self.experiment_id},equipment_id={self.equipment_id},)" @@ -200,8 +95,8 @@ class ExperimentMethod(Base): __tablename__ = 'experiment_method' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - method_id = Column(Integer()) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + method_id = Column(Integer(), ForeignKey('Method.id')) def __repr__(self): @@ -212,57 +107,42 @@ def __repr__(self): -class ExperimentEquipment(Base): - """ - Link between Experiment and Equipment. - """ - __tablename__ = 'experiment_equipment' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - equipment_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentEquipment(id={self.id},experiment_id={self.experiment_id},equipment_id={self.equipment_id},)" - - - - - - -class ExperimentAnalysis(Base): +class ExperimentPreparedSample(Base): """ - Link between Experiment and AnalysisType. + Link between Experiment and PreparedSample. """ - __tablename__ = 'experiment_analysis' + __tablename__ = 'experiment_prepared_sample' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - analysis_type_id = Column(Integer()) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + prepared_sample_id = Column(Integer(), ForeignKey('PreparedSample.id')) def __repr__(self): - return f"ExperimentAnalysis(id={self.id},experiment_id={self.experiment_id},analysis_type_id={self.analysis_type_id},)" + return f"ExperimentPreparedSample(id={self.id},experiment_id={self.experiment_id},prepared_sample_id={self.prepared_sample_id},)" -class ExperimentPreparedSample(Base): +class Geography(Base): """ - Link between Experiment and PreparedSample. + Geographic location. """ - __tablename__ = 'experiment_prepared_sample' + __tablename__ = 'geography' - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - prepared_sample_id = Column(Integer()) + geoid = Column(Text(), primary_key=True, nullable=False ) + state_name = Column(Text()) + state_fips = Column(Text()) + county_name = Column(Text()) + county_fips = Column(Text()) + region_name = Column(Text()) + agg_level_desc = Column(Text()) def __repr__(self): - return f"ExperimentPreparedSample(id={self.id},experiment_id={self.experiment_id},prepared_sample_id={self.prepared_sample_id},)" + return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" @@ -407,89 +287,192 @@ def __repr__(self): -class InfrastructureEthanolBiorefineries(Base): +class InfrastructureCombustionPlants(Base): """ - Ethanol biorefineries infrastructure. + Combustion plants infrastructure. """ - __tablename__ = 'infrastructure_ethanol_biorefineries' + __tablename__ = 'infrastructure_combustion_plants' - ethanol_biorefinery_id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) + combustion_fid = Column(Integer(), primary_key=True, nullable=False ) + objectid = Column(Integer()) + status = Column(Text()) city = Column(Text()) - state = Column(Text()) - address = Column(Text()) + name = Column(Text()) + county = Column(Text()) + equivalent_generation = Column(Numeric()) + np_mw = Column(Numeric()) + cf = Column(Numeric()) + yearload = Column(Integer()) + fuel = Column(Text()) + notes = Column(Text()) type = Column(Text()) - capacity_mgy = Column(Integer()) - production_mgy = Column(Integer()) - constr_exp = Column(Integer()) + wkt_geom = Column(Text()) + geom = Column(Text()) + latitude = Column(Numeric()) + longitude = Column(Numeric()) def __repr__(self): - return f"InfrastructureEthanolBiorefineries(ethanol_biorefinery_id={self.ethanol_biorefinery_id},name={self.name},city={self.city},state={self.state},address={self.address},type={self.type},capacity_mgy={self.capacity_mgy},production_mgy={self.production_mgy},constr_exp={self.constr_exp},)" + return f"InfrastructureCombustionPlants(combustion_fid={self.combustion_fid},objectid={self.objectid},status={self.status},city={self.city},name={self.name},county={self.county},equivalent_generation={self.equivalent_generation},np_mw={self.np_mw},cf={self.cf},yearload={self.yearload},fuel={self.fuel},notes={self.notes},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" -class InfrastructureLandfills(Base): +class InfrastructureDistrictEnergySystems(Base): """ - Landfills infrastructure. + District energy systems infrastructure. """ - __tablename__ = 'infrastructure_landfills' + __tablename__ = 'infrastructure_district_energy_systems' - project_id = Column(Text(), primary_key=True, nullable=False ) - project_int_id = Column(Integer()) - ghgrp_id = Column(Text()) - landfill_id = Column(Integer()) - landfill_name = Column(Text()) - state = Column(Text()) - physical_address = Column(Text()) + des_fid = Column(Integer(), primary_key=True, nullable=False ) + cbg_id = Column(Integer()) + name = Column(Text()) + system = Column(Text()) + object_id = Column(Integer()) city = Column(Text()) - county = Column(Text()) - zip_code = Column(Text()) + state = Column(Text()) + primary_fuel = Column(Text()) + secondary_fuel = Column(Text()) + usetype = Column(Text()) + cap_st = Column(Numeric()) + cap_hw = Column(Numeric()) + cap_cw = Column(Numeric()) + chpcg_cap = Column(Numeric()) + excess_c = Column(Numeric()) + excess_h = Column(Numeric()) + type = Column(Text()) + wkt_geom = Column(Text()) + geom = Column(Text()) latitude = Column(Numeric()) longitude = Column(Numeric()) - ownership_type = Column(Text()) - landfill_owner_orgs = Column(Text()) - landfill_opened_year = Column(Date()) - landfill_closure_year = Column(Date()) - landfill_status = Column(Text()) - waste_in_place = Column(Integer()) - waste_in_place_year = Column(Date()) - lfg_system_in_place = Column(Boolean()) - lfg_collected = Column(Numeric()) - lfg_flared = Column(Numeric()) - project_status = Column(Text()) - project_name = Column(Text()) - project_start_date = Column(Date()) - project_shutdown_date = Column(Date()) - project_type_category = Column(Text()) - lfg_energy_project_type = Column(Text()) - rng_delivery_method = Column(Text()) - actual_mw_generation = Column(Numeric()) - rated_mw_capacity = Column(Numeric()) - lfg_flow_to_project = Column(Numeric()) - direct_emission_reductions = Column(Numeric()) - avoided_emission_reductions = Column(Numeric()) def __repr__(self): - return f"InfrastructureLandfills(project_id={self.project_id},project_int_id={self.project_int_id},ghgrp_id={self.ghgrp_id},landfill_id={self.landfill_id},landfill_name={self.landfill_name},state={self.state},physical_address={self.physical_address},city={self.city},county={self.county},zip_code={self.zip_code},latitude={self.latitude},longitude={self.longitude},ownership_type={self.ownership_type},landfill_owner_orgs={self.landfill_owner_orgs},landfill_opened_year={self.landfill_opened_year},landfill_closure_year={self.landfill_closure_year},landfill_status={self.landfill_status},waste_in_place={self.waste_in_place},waste_in_place_year={self.waste_in_place_year},lfg_system_in_place={self.lfg_system_in_place},lfg_collected={self.lfg_collected},lfg_flared={self.lfg_flared},project_status={self.project_status},project_name={self.project_name},project_start_date={self.project_start_date},project_shutdown_date={self.project_shutdown_date},project_type_category={self.project_type_category},lfg_energy_project_type={self.lfg_energy_project_type},rng_delivery_method={self.rng_delivery_method},actual_mw_generation={self.actual_mw_generation},rated_mw_capacity={self.rated_mw_capacity},lfg_flow_to_project={self.lfg_flow_to_project},direct_emission_reductions={self.direct_emission_reductions},avoided_emission_reductions={self.avoided_emission_reductions},)" + return f"InfrastructureDistrictEnergySystems(des_fid={self.des_fid},cbg_id={self.cbg_id},name={self.name},system={self.system},object_id={self.object_id},city={self.city},state={self.state},primary_fuel={self.primary_fuel},secondary_fuel={self.secondary_fuel},usetype={self.usetype},cap_st={self.cap_st},cap_hw={self.cap_hw},cap_cw={self.cap_cw},chpcg_cap={self.chpcg_cap},excess_c={self.excess_c},excess_h={self.excess_h},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" -class InfrastructureLivestockAnaerobicDigesters(Base): +class InfrastructureEthanolBiorefineries(Base): """ - Livestock anaerobic digesters infrastructure. + Ethanol biorefineries infrastructure. """ - __tablename__ = 'infrastructure_livestock_anaerobic_digesters' + __tablename__ = 'infrastructure_ethanol_biorefineries' - digester_id = Column(Integer(), primary_key=True, nullable=False ) - project_name = Column(Text()) + ethanol_biorefinery_id = Column(Integer(), primary_key=True, nullable=False ) + name = Column(Text()) + city = Column(Text()) + state = Column(Text()) + address = Column(Text()) + type = Column(Text()) + capacity_mgy = Column(Integer()) + production_mgy = Column(Integer()) + constr_exp = Column(Integer()) + + + def __repr__(self): + return f"InfrastructureEthanolBiorefineries(ethanol_biorefinery_id={self.ethanol_biorefinery_id},name={self.name},city={self.city},state={self.state},address={self.address},type={self.type},capacity_mgy={self.capacity_mgy},production_mgy={self.production_mgy},constr_exp={self.constr_exp},)" + + + + + + +class InfrastructureFoodProcessingFacilities(Base): + """ + Food processing facilities infrastructure. + """ + __tablename__ = 'infrastructure_food_processing_facilities' + + processing_facility_id = Column(Integer(), primary_key=True, nullable=False ) + address = Column(Text()) + county = Column(Text()) + city = Column(Text()) + company = Column(Text()) + join_count = Column(Integer()) + master_type = Column(Text()) + state = Column(Text()) + subtype = Column(Text()) + target_fid = Column(Integer()) + processing_type = Column(Text()) + zip = Column(Text()) + type = Column(Text()) + wkt_geom = Column(Text()) + geom = Column(Text()) + latitude = Column(Numeric()) + longitude = Column(Numeric()) + + + def __repr__(self): + return f"InfrastructureFoodProcessingFacilities(processing_facility_id={self.processing_facility_id},address={self.address},county={self.county},city={self.city},company={self.company},join_count={self.join_count},master_type={self.master_type},state={self.state},subtype={self.subtype},target_fid={self.target_fid},processing_type={self.processing_type},zip={self.zip},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + + + + + + +class InfrastructureLandfills(Base): + """ + Landfills infrastructure. + """ + __tablename__ = 'infrastructure_landfills' + + project_id = Column(Text(), primary_key=True, nullable=False ) + project_int_id = Column(Integer()) + ghgrp_id = Column(Text()) + landfill_id = Column(Integer()) + landfill_name = Column(Text()) + state = Column(Text()) + physical_address = Column(Text()) + city = Column(Text()) + county = Column(Text()) + zip_code = Column(Text()) + latitude = Column(Numeric()) + longitude = Column(Numeric()) + ownership_type = Column(Text()) + landfill_owner_orgs = Column(Text()) + landfill_opened_year = Column(Date()) + landfill_closure_year = Column(Date()) + landfill_status = Column(Text()) + waste_in_place = Column(Integer()) + waste_in_place_year = Column(Date()) + lfg_system_in_place = Column(Boolean()) + lfg_collected = Column(Numeric()) + lfg_flared = Column(Numeric()) + project_status = Column(Text()) + project_name = Column(Text()) + project_start_date = Column(Date()) + project_shutdown_date = Column(Date()) + project_type_category = Column(Text()) + lfg_energy_project_type = Column(Text()) + rng_delivery_method = Column(Text()) + actual_mw_generation = Column(Numeric()) + rated_mw_capacity = Column(Numeric()) + lfg_flow_to_project = Column(Numeric()) + direct_emission_reductions = Column(Numeric()) + avoided_emission_reductions = Column(Numeric()) + + + def __repr__(self): + return f"InfrastructureLandfills(project_id={self.project_id},project_int_id={self.project_int_id},ghgrp_id={self.ghgrp_id},landfill_id={self.landfill_id},landfill_name={self.landfill_name},state={self.state},physical_address={self.physical_address},city={self.city},county={self.county},zip_code={self.zip_code},latitude={self.latitude},longitude={self.longitude},ownership_type={self.ownership_type},landfill_owner_orgs={self.landfill_owner_orgs},landfill_opened_year={self.landfill_opened_year},landfill_closure_year={self.landfill_closure_year},landfill_status={self.landfill_status},waste_in_place={self.waste_in_place},waste_in_place_year={self.waste_in_place_year},lfg_system_in_place={self.lfg_system_in_place},lfg_collected={self.lfg_collected},lfg_flared={self.lfg_flared},project_status={self.project_status},project_name={self.project_name},project_start_date={self.project_start_date},project_shutdown_date={self.project_shutdown_date},project_type_category={self.project_type_category},lfg_energy_project_type={self.lfg_energy_project_type},rng_delivery_method={self.rng_delivery_method},actual_mw_generation={self.actual_mw_generation},rated_mw_capacity={self.rated_mw_capacity},lfg_flow_to_project={self.lfg_flow_to_project},direct_emission_reductions={self.direct_emission_reductions},avoided_emission_reductions={self.avoided_emission_reductions},)" + + + + + + +class InfrastructureLivestockAnaerobicDigesters(Base): + """ + Livestock anaerobic digesters infrastructure. + """ + __tablename__ = 'infrastructure_livestock_anaerobic_digesters' + + digester_id = Column(Integer(), primary_key=True, nullable=False ) + project_name = Column(Text()) project_type = Column(Text()) city = Column(Text()) state = Column(Text()) @@ -521,6 +504,38 @@ def __repr__(self): +class InfrastructureMswToEnergyAnaerobicDigesters(Base): + """ + MSW to energy anaerobic digesters infrastructure. + """ + __tablename__ = 'infrastructure_msw_to_energy_anaerobic_digesters' + + wte_id = Column(Integer(), primary_key=True, nullable=False ) + city = Column(Text()) + county = Column(Text()) + equivalent_generation = Column(Numeric()) + feedstock = Column(Text()) + dayload = Column(Numeric()) + dayload_bdt = Column(Numeric()) + facility_type = Column(Text()) + status = Column(Text()) + notes = Column(Text()) + source = Column(Text()) + type = Column(Text()) + wkt_geom = Column(Text()) + geom = Column(Text()) + latitude = Column(Numeric()) + longitude = Column(Numeric()) + + + def __repr__(self): + return f"InfrastructureMswToEnergyAnaerobicDigesters(wte_id={self.wte_id},city={self.city},county={self.county},equivalent_generation={self.equivalent_generation},feedstock={self.feedstock},dayload={self.dayload},dayload_bdt={self.dayload_bdt},facility_type={self.facility_type},status={self.status},notes={self.notes},source={self.source},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + + + + + + class InfrastructureSafAndRenewableDieselPlants(Base): """ SAF and renewable diesel plants infrastructure. @@ -596,153 +611,152 @@ def __repr__(self): -class InfrastructureCombustionPlants(Base): +class LineageGroup(Base): """ - Combustion plants infrastructure. + Grouping for lineage information. """ - __tablename__ = 'infrastructure_combustion_plants' + __tablename__ = 'lineage_group' - combustion_fid = Column(Integer(), primary_key=True, nullable=False ) - objectid = Column(Integer()) + id = Column(Integer(), primary_key=True, nullable=False ) + etl_run_id = Column(Text()) + note = Column(Text()) + + + def __repr__(self): + return f"LineageGroup(id={self.id},etl_run_id={self.etl_run_id},note={self.note},)" + + + + + + +class EntityLineage(Base): + """ + Lineage information for a specific entity. + """ + __tablename__ = 'entity_lineage' + + id = Column(Integer(), primary_key=True, nullable=False ) + lineage_group_id = Column(Integer()) + source_table = Column(Text()) + source_row_id = Column(Text()) + note = Column(Text()) + + + def __repr__(self): + return f"EntityLineage(id={self.id},lineage_group_id={self.lineage_group_id},source_table={self.source_table},source_row_id={self.source_row_id},note={self.note},)" + + + + + + +class EtlRun(Base): + """ + Information about an ETL run. + """ + __tablename__ = 'etl_run' + + id = Column(Integer(), primary_key=True, nullable=False ) + started_at = Column(DateTime()) + completed_at = Column(DateTime()) + pipeline_name = Column(Text()) status = Column(Text()) - city = Column(Text()) - name = Column(Text()) - county = Column(Text()) - equivalent_generation = Column(Numeric()) - np_mw = Column(Numeric()) - cf = Column(Numeric()) - yearload = Column(Integer()) - fuel = Column(Text()) - notes = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) + records_ingested = Column(Integer()) + note = Column(Text()) def __repr__(self): - return f"InfrastructureCombustionPlants(combustion_fid={self.combustion_fid},objectid={self.objectid},status={self.status},city={self.city},name={self.name},county={self.county},equivalent_generation={self.equivalent_generation},np_mw={self.np_mw},cf={self.cf},yearload={self.yearload},fuel={self.fuel},notes={self.notes},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + return f"EtlRun(id={self.id},started_at={self.started_at},completed_at={self.completed_at},pipeline_name={self.pipeline_name},status={self.status},records_ingested={self.records_ingested},note={self.note},)" -class InfrastructureDistrictEnergySystems(Base): +class ParameterCategoryParameter(Base): """ - District energy systems infrastructure. + Link between Parameter and ParameterCategory. """ - __tablename__ = 'infrastructure_district_energy_systems' + __tablename__ = 'parameter_category_parameter' - des_fid = Column(Integer(), primary_key=True, nullable=False ) - cbg_id = Column(Integer()) - name = Column(Text()) - system = Column(Text()) - object_id = Column(Integer()) - city = Column(Text()) - state = Column(Text()) - primary_fuel = Column(Text()) - secondary_fuel = Column(Text()) - usetype = Column(Text()) - cap_st = Column(Numeric()) - cap_hw = Column(Numeric()) - cap_cw = Column(Numeric()) - chpcg_cap = Column(Numeric()) - excess_c = Column(Numeric()) - excess_h = Column(Numeric()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer(), ForeignKey('Parameter.id')) + parameter_category_id = Column(Integer(), ForeignKey('ParameterCategory.id')) def __repr__(self): - return f"InfrastructureDistrictEnergySystems(des_fid={self.des_fid},cbg_id={self.cbg_id},name={self.name},system={self.system},object_id={self.object_id},city={self.city},state={self.state},primary_fuel={self.primary_fuel},secondary_fuel={self.secondary_fuel},usetype={self.usetype},cap_st={self.cap_st},cap_hw={self.cap_hw},cap_cw={self.cap_cw},chpcg_cap={self.chpcg_cap},excess_c={self.excess_c},excess_h={self.excess_h},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" -class InfrastructureFoodProcessingFacilities(Base): +class ParameterUnit(Base): """ - Food processing facilities infrastructure. + Link between Parameter and Unit (alternate units). """ - __tablename__ = 'infrastructure_food_processing_facilities' + __tablename__ = 'parameter_unit' - processing_facility_id = Column(Integer(), primary_key=True, nullable=False ) - address = Column(Text()) - county = Column(Text()) - city = Column(Text()) - company = Column(Text()) - join_count = Column(Integer()) - master_type = Column(Text()) - state = Column(Text()) - subtype = Column(Text()) - target_fid = Column(Integer()) - processing_type = Column(Text()) - zip = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) + id = Column(Integer(), primary_key=True, nullable=False ) + parameter_id = Column(Integer(), ForeignKey('Parameter.id')) + alternate_unit_id = Column(Integer(), ForeignKey('Unit.id')) def __repr__(self): - return f"InfrastructureFoodProcessingFacilities(processing_facility_id={self.processing_facility_id},address={self.address},county={self.county},city={self.city},company={self.company},join_count={self.join_count},master_type={self.master_type},state={self.state},subtype={self.subtype},target_fid={self.target_fid},processing_type={self.processing_type},zip={self.zip},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" -class InfrastructureMswToEnergyAnaerobicDigesters(Base): +class ResourceMorphology(Base): """ - MSW to energy anaerobic digesters infrastructure. + Morphology of a resource. """ - __tablename__ = 'infrastructure_msw_to_energy_anaerobic_digesters' + __tablename__ = 'resource_morphology' - wte_id = Column(Integer(), primary_key=True, nullable=False ) - city = Column(Text()) - county = Column(Text()) - equivalent_generation = Column(Numeric()) - feedstock = Column(Text()) - dayload = Column(Numeric()) - dayload_bdt = Column(Numeric()) - facility_type = Column(Text()) - status = Column(Text()) - notes = Column(Text()) - source = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) + id = Column(Integer(), primary_key=True, nullable=False ) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + morphology_uri = Column(Text()) def __repr__(self): - return f"InfrastructureMswToEnergyAnaerobicDigesters(wte_id={self.wte_id},city={self.city},county={self.county},equivalent_generation={self.equivalent_generation},feedstock={self.feedstock},dayload={self.dayload},dayload_bdt={self.dayload_bdt},facility_type={self.facility_type},status={self.status},notes={self.notes},source={self.source},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" + return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" -class Resource(BaseEntity): +class DataSource(BaseEntity): """ - Biomass resource definition. + Source of data. """ - __tablename__ = 'resource' + __tablename__ = 'data_source' name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) + description = Column(Text()) + data_source_type_id = Column(Integer(), ForeignKey('DataSourceType.id')) + full_title = Column(Text()) + creator = Column(Text()) + subject = Column(Text()) + publisher = Column(Text()) + contributor = Column(Text()) + date = Column(DateTime()) + type = Column(Text()) + biocirv = Column(Boolean()) + format = Column(Text()) + language = Column(Text()) + relation = Column(Text()) + temporal_coverage = Column(Text()) + location_coverage_id = Column(Integer(), ForeignKey('LocationResolution.id')) + rights = Column(Text()) + license = Column(Text()) + uri = Column(Text()) note = Column(Text()) - test = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -751,7 +765,7 @@ class Resource(BaseEntity): def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -763,20 +777,22 @@ def __repr__(self): -class ResourceClass(LookupBase): +class DataSourceType(BaseEntity): """ - Classification of resources. + Type of data source (e.g. database, literature). """ - __tablename__ = 'resource_class' + __tablename__ = 'data_source_type' + source_type_id = Column(Integer(), ForeignKey('SourceType.id')) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -788,11 +804,43 @@ def __repr__(self): -class ResourceSubclass(LookupBase): +class FileObjectMetadata(BaseEntity): """ - Sub-classification of resources. + Metadata for a file object. """ - __tablename__ = 'resource_subclass' + __tablename__ = 'file_object_metadata' + + data_source_id = Column(Integer(), ForeignKey('DataSource.id')) + bucket_path = Column(Text()) + file_format = Column(Text()) + file_size = Column(Integer()) + checksum_md5 = Column(Text()) + checksum_sha256 = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) + + + def __repr__(self): + return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + + + + + # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html + __mapper_args__ = { + 'concrete': True + } + + + +class LocationResolution(LookupBase): + """ + Resolution of the location (e.g. nation, state, county). + """ + __tablename__ = 'location_resolution' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -801,7 +849,7 @@ class ResourceSubclass(LookupBase): def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -813,13 +861,12 @@ def __repr__(self): -class PrimaryAgProduct(LookupBase): +class SourceType(LookupBase): """ - Primary agricultural product definition. + Type of source (e.g. database, literature). """ - __tablename__ = 'primary_ag_product' + __tablename__ = 'source_type' - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) description = Column(Text()) @@ -827,7 +874,7 @@ class PrimaryAgProduct(LookupBase): def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -839,27 +886,21 @@ def __repr__(self): -class ResourceAvailability(BaseEntity): +class Equipment(LookupBase): """ - Availability of a resource in a location. + Equipment used in experiments. """ - __tablename__ = 'resource_availability' + __tablename__ = 'equipment' - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) + equipment_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Equipment(equipment_location_id={self.equipment_location_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -871,29 +912,18 @@ def __repr__(self): -class ResourceCounterfactual(BaseEntity): +class Experiment(BaseEntity): """ - Counterfactual uses of a resource. + Experiment definition. """ - __tablename__ = 'resource_counterfactual' + __tablename__ = 'experiment' - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + exper_start_date = Column(Date()) + exper_duration = Column(Numeric()) + exper_duration_unit_id = Column(Integer(), ForeignKey('Unit.id')) + exper_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + description = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -902,7 +932,7 @@ class ResourceCounterfactual(BaseEntity): def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Experiment(analyst_id={self.analyst_id},exper_start_date={self.exper_start_date},exper_duration={self.exper_duration},exper_duration_unit_id={self.exper_duration_unit_id},exper_location_id={self.exper_location_id},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -914,39 +944,20 @@ def __repr__(self): -class FieldSample(BaseEntity): +class AnalysisType(LookupBase): """ - Sample collected from the field. + Type of analysis. """ - __tablename__ = 'field_sample' + __tablename__ = 'analysis_type' - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -958,17 +969,19 @@ def __repr__(self): -class PhysicalCharacteristic(BaseEntity): +class Method(BaseEntity): """ - Physical characteristics of a sample. + method. """ - __tablename__ = 'physical_characteristic' + __tablename__ = 'method' - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) + name = Column(Text()) + method_abbrev_id = Column(Integer(), ForeignKey('MethodAbbrev.id')) + method_category_id = Column(Integer(), ForeignKey('MethodCategory.id')) + method_standard_id = Column(Integer(), ForeignKey('MethodStandard.id')) + description = Column(Text()) + detection_limits = Column(Text()) + source_id = Column(Integer(), ForeignKey('DataSource.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -977,7 +990,7 @@ class PhysicalCharacteristic(BaseEntity): def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -989,17 +1002,18 @@ def __repr__(self): -class FieldSampleCondition(BaseEntity): +class PreparedSample(BaseEntity): """ - Condition of the field sample. + Sample that has been prepared. """ - __tablename__ = 'field_sample_condition' + __tablename__ = 'prepared_sample' - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) + name = Column(Text()) + field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) + prep_method_id = Column(Integer(), ForeignKey('PreparationMethod.id')) + prep_date = Column(Date()) + preparer_id = Column(Integer(), ForeignKey('Contact.id')) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1008,7 +1022,7 @@ class FieldSampleCondition(BaseEntity): def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1020,20 +1034,27 @@ def __repr__(self): -class FieldStorageMethod(LookupBase): +class Resource(BaseEntity): """ - Method of field storage. + Biomass resource definition. """ - __tablename__ = 'field_storage_method' + __tablename__ = 'resource' - id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + primary_ag_product_id = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + resource_class_id = Column(Integer(), ForeignKey('ResourceClass.id')) + resource_subclass_id = Column(Integer(), ForeignKey('ResourceSubclass.id')) + note = Column(Text()) + test = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1045,11 +1066,11 @@ def __repr__(self): -class CollectionMethod(LookupBase): +class ResourceSubclass(LookupBase): """ - Method of collection. + Sub-classification of resources. """ - __tablename__ = 'collection_method' + __tablename__ = 'resource_subclass' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1058,7 +1079,7 @@ class CollectionMethod(LookupBase): def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1070,11 +1091,11 @@ def __repr__(self): -class HarvestMethod(LookupBase): +class Unit(LookupBase): """ - Method of harvest. + Unit of measurement. """ - __tablename__ = 'harvest_method' + __tablename__ = 'unit' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1083,7 +1104,7 @@ class HarvestMethod(LookupBase): def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1095,20 +1116,35 @@ def __repr__(self): -class ProcessingMethod(LookupBase): +class BillionTon2023Record(BaseEntity): """ - Method of processing. + Billion Ton 2023 record. """ - __tablename__ = 'processing_method' + __tablename__ = 'billion_ton2023_record' + subclass_id = Column(Integer(), ForeignKey('ResourceSubclass.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + county_square_miles = Column(Float()) + model_name = Column(Text()) + scenario_name = Column(Text()) + price_offered_usd = Column(Numeric()) + production = Column(Integer()) + production_unit_id = Column(Integer(), ForeignKey('Unit.id')) + btu_ton = Column(Integer()) + production_energy_content = Column(Integer()) + energy_content_unit_id = Column(Integer(), ForeignKey('Unit.id')) + product_density_dtpersqmi = Column(Numeric()) + land_source = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"BillionTon2023Record(subclass_id={self.subclass_id},resource_id={self.resource_id},geoid={self.geoid},county_square_miles={self.county_square_miles},model_name={self.model_name},scenario_name={self.scenario_name},price_offered_usd={self.price_offered_usd},production={self.production},production_unit_id={self.production_unit_id},btu_ton={self.btu_ton},production_energy_content={self.production_energy_content},energy_content_unit_id={self.energy_content_unit_id},product_density_dtpersqmi={self.product_density_dtpersqmi},land_source={self.land_source},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1120,11 +1156,11 @@ def __repr__(self): -class SoilType(LookupBase): +class PrimaryAgProduct(LookupBase): """ - Type of soil. + Primary agricultural product definition. """ - __tablename__ = 'soil_type' + __tablename__ = 'primary_ag_product' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1133,7 +1169,7 @@ class SoilType(LookupBase): def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PrimaryAgProduct(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1145,20 +1181,32 @@ def __repr__(self): -class AgTreatment(LookupBase): +class LandiqRecord(BaseEntity): """ - Agricultural treatment. + LandIQ record. """ - __tablename__ = 'ag_treatment' + __tablename__ = 'landiq_record' + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + polygon_id = Column(Integer(), ForeignKey('Polygon.id')) + main_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + secondary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + tertiary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + quaternary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + confidence = Column(Integer()) + irrigated = Column(Boolean()) + acres = Column(Float()) + version = Column(Text()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"LandiqRecord(dataset_id={self.dataset_id},polygon_id={self.polygon_id},main_crop={self.main_crop},secondary_crop={self.secondary_crop},tertiary_crop={self.tertiary_crop},quaternary_crop={self.quaternary_crop},confidence={self.confidence},irrigated={self.irrigated},acres={self.acres},version={self.version},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1170,23 +1218,23 @@ def __repr__(self): -class LocationSoilType(BaseEntity): +class UsdaCommodity(LookupBase): """ - Soil type at a location. + USDA commodity. """ - __tablename__ = 'location_soil_type' + __tablename__ = 'usda_commodity' - location_id = Column(Integer()) - soil_type_id = Column(Integer()) + usda_source = Column(Text()) + usda_code = Column(Text()) + parent_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaCommodity(usda_source={self.usda_source},usda_code={self.usda_code},parent_commodity_id={self.parent_commodity_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1198,20 +1246,17 @@ def __repr__(self): -class LocationAddress(BaseEntity): +class ResourceUsdaCommodityMap(BaseEntity): """ - Physical address. + Mapping between resources/crops and USDA commodities. """ - __tablename__ = 'location_address' + __tablename__ = 'resource_usda_commodity_map' - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + primary_ag_product_id = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + usda_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + match_tier = Column(Text()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1220,7 +1265,7 @@ class LocationAddress(BaseEntity): def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1232,16 +1277,18 @@ def __repr__(self): -class Contact(BaseEntity): +class UsdaCensusRecord(BaseEntity): """ - Contact information for a person. + USDA census record. """ - __tablename__ = 'contact' + __tablename__ = 'usda_census_record' - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + geoid = Column(Text()) + commodity_code = Column(Integer(), ForeignKey('UsdaCommodity.id')) + year = Column(Integer()) + source_reference = Column(Text()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1250,7 +1297,7 @@ class Contact(BaseEntity): def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaCensusRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},source_reference={self.source_reference},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1262,22 +1309,20 @@ def __repr__(self): -class Provider(BaseEntity): +class UsdaDomain(LookupBase): """ - Provider information. + USDA domain. """ - __tablename__ = 'provider' + __tablename__ = 'usda_domain' - codename = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaDomain(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1289,32 +1334,20 @@ def __repr__(self): -class DataSource(BaseEntity): +class UsdaMarketReport(BaseEntity): """ - Source of data. + USDA market report. """ - __tablename__ = 'data_source' + __tablename__ = 'usda_market_report' - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) + slug_id = Column(Integer()) + slug_name = Column(Text()) + report_series_title = Column(Text()) + frequency = Column(Text()) + office_name = Column(Text()) + office_city_id = Column(Integer(), ForeignKey('LocationAddress.id')) + office_state_fips = Column(Text()) + source_id = Column(Integer(), ForeignKey('DataSource.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1323,7 +1356,7 @@ class DataSource(BaseEntity): def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaMarketReport(slug_id={self.slug_id},slug_name={self.slug_name},report_series_title={self.report_series_title},frequency={self.frequency},office_name={self.office_name},office_city_id={self.office_city_id},office_state_fips={self.office_state_fips},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1335,18 +1368,32 @@ def __repr__(self): -class FileObjectMetadata(BaseEntity): +class UsdaMarketRecord(BaseEntity): """ - Metadata for a file object. + USDA market record. """ - __tablename__ = 'file_object_metadata' + __tablename__ = 'usda_market_record' - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) + report_id = Column(Integer(), ForeignKey('UsdaMarketReport.id')) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + report_begin_date = Column(DateTime()) + report_end_date = Column(DateTime()) + report_date = Column(DateTime()) + commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + market_type_id = Column(Integer()) + market_type_category = Column(Text()) + grp = Column(Text()) + market_category_id = Column(Integer()) + class_ = Column(Text()) + grade = Column(Text()) + variety = Column(Text()) + protein_pct = Column(Numeric()) + application = Column(Text()) + pkg = Column(Text()) + sale_type = Column(Text()) + price_unit_id = Column(Integer(), ForeignKey('Unit.id')) + freight = Column(Text()) + trans_mode = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1355,7 +1402,7 @@ class FileObjectMetadata(BaseEntity): def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaMarketRecord(report_id={self.report_id},dataset_id={self.dataset_id},report_begin_date={self.report_begin_date},report_end_date={self.report_end_date},report_date={self.report_date},commodity_id={self.commodity_id},market_type_id={self.market_type_id},market_type_category={self.market_type_category},grp={self.grp},market_category_id={self.market_category_id},class_={self.class_},grade={self.grade},variety={self.variety},protein_pct={self.protein_pct},application={self.application},pkg={self.pkg},sale_type={self.sale_type},price_unit_id={self.price_unit_id},freight={self.freight},trans_mode={self.trans_mode},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1367,13 +1414,20 @@ def __repr__(self): -class DataSourceType(BaseEntity): +class LocationAddress(BaseEntity): """ - Type of data source (e.g. database, literature). + Physical address. """ - __tablename__ = 'data_source_type' + __tablename__ = 'location_address' - source_type_id = Column(Integer()) + geography_id = Column(Text(), ForeignKey('Geography.geoid')) + address_line1 = Column(Text()) + address_line2 = Column(Text()) + city = Column(Text()) + zip = Column(Text()) + lat = Column(Float()) + lon = Column(Float()) + is_anonymous = Column(Boolean()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1382,7 +1436,7 @@ class DataSourceType(BaseEntity): def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1394,11 +1448,11 @@ def __repr__(self): -class LocationResolution(LookupBase): +class UsdaStatisticCategory(LookupBase): """ - Resolution of the location (e.g. nation, state, county). + USDA statistic category. """ - __tablename__ = 'location_resolution' + __tablename__ = 'usda_statistic_category' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1407,7 +1461,7 @@ class LocationResolution(LookupBase): def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"UsdaStatisticCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1419,11 +1473,11 @@ def __repr__(self): -class SourceType(LookupBase): +class UsdaSurveyProgram(LookupBase): """ - Type of source (e.g. database, literature). + USDA survey program. """ - __tablename__ = 'source_type' + __tablename__ = 'usda_survey_program' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1432,7 +1486,7 @@ class SourceType(LookupBase): def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"UsdaSurveyProgram(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1444,18 +1498,21 @@ def __repr__(self): -class PreparationMethod(BaseEntity): +class UsdaSurveyRecord(BaseEntity): """ - Method of sample preparation. + USDA survey record. """ - __tablename__ = 'preparation_method' + __tablename__ = 'usda_survey_record' - name = Column(Text()) - description = Column(Text()) - prep_method_abbrev_id = Column(Integer()) - prep_temp_c = Column(Numeric()) - uri = Column(Text()) - drying_step = Column(Boolean()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + geoid = Column(Text()) + commodity_code = Column(Integer(), ForeignKey('UsdaCommodity.id')) + year = Column(Integer()) + survey_program_id = Column(Integer(), ForeignKey('UsdaSurveyProgram.id')) + survey_period = Column(Text()) + reference_month = Column(Text()) + seasonal_flag = Column(Boolean()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1464,7 +1521,7 @@ class PreparationMethod(BaseEntity): def __repr__(self): - return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"UsdaSurveyRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},survey_program_id={self.survey_program_id},survey_period={self.survey_period},reference_month={self.reference_month},seasonal_flag={self.seasonal_flag},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1476,20 +1533,27 @@ def __repr__(self): -class PreparationMethodAbbreviation(LookupBase): +class UsdaTermMap(BaseEntity): """ - Abbreviation for preparation method. + Mapping of raw terms to USDA commodities. """ - __tablename__ = 'preparation_method_abbreviation' + __tablename__ = 'usda_term_map' + source_system = Column(Text()) + source_context = Column(Text()) + raw_term = Column(Text()) + usda_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + is_verified = Column(Boolean()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"UsdaTermMap(source_system={self.source_system},source_context={self.source_context},raw_term={self.raw_term},usda_commodity_id={self.usda_commodity_id},is_verified={self.is_verified},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1501,27 +1565,20 @@ def __repr__(self): -class PreparedSample(BaseEntity): +class AgTreatment(LookupBase): """ - Sample that has been prepared. + Agricultural treatment. """ - __tablename__ = 'prepared_sample' + __tablename__ = 'ag_treatment' - name = Column(Text()) - field_sample_id = Column(Integer()) - prep_method_id = Column(Integer()) - prep_date = Column(Date()) - preparer_id = Column(Integer()) - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1533,11 +1590,11 @@ def __repr__(self): -class Unit(LookupBase): +class CollectionMethod(LookupBase): """ - Unit of measurement. + Method of collection. """ - __tablename__ = 'unit' + __tablename__ = 'collection_method' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1546,7 +1603,7 @@ class Unit(LookupBase): def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1558,19 +1615,30 @@ def __repr__(self): -class Method(BaseEntity): +class FieldSample(BaseEntity): """ - Analytical method. + Sample collected from the field. """ - __tablename__ = 'method' + __tablename__ = 'field_sample' name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + provider_id = Column(Integer(), ForeignKey('Provider.id')) + collector_id = Column(Integer(), ForeignKey('Contact.id')) + sample_collection_source = Column(Text()) + amount_collected = Column(Numeric()) + amount_collected_unit_id = Column(Integer(), ForeignKey('Unit.id')) + sampling_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + field_storage_method_id = Column(Integer(), ForeignKey('FieldStorageMethod.id')) + field_storage_duration_value = Column(Numeric()) + field_storage_duration_unit_id = Column(Integer(), ForeignKey('Unit.id')) + field_storage_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + collection_timestamp = Column(DateTime()) + collection_method_id = Column(Integer(), ForeignKey('CollectionMethod.id')) + harvest_method_id = Column(Integer(), ForeignKey('HarvestMethod.id')) + harvest_date = Column(Date()) + field_sample_storage_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1579,7 +1647,7 @@ class Method(BaseEntity): def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1591,11 +1659,11 @@ def __repr__(self): -class MethodAbbrev(LookupBase): +class ProcessingMethod(LookupBase): """ - Abbreviation for method. + Method of processing. """ - __tablename__ = 'method_abbrev' + __tablename__ = 'processing_method' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1604,7 +1672,7 @@ class MethodAbbrev(LookupBase): def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1616,20 +1684,26 @@ def __repr__(self): -class MethodCategory(LookupBase): +class FieldSampleCondition(BaseEntity): """ - Category of method. + Condition of the field sample. """ - __tablename__ = 'method_category' + __tablename__ = 'field_sample_condition' + field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) + ag_treatment_id = Column(Integer(), ForeignKey('AgTreatment.id')) + last_application_date = Column(Date()) + treatment_amount_per_acre = Column(Float()) + processing_method_id = Column(Integer(), ForeignKey('ProcessingMethod.id')) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1641,11 +1715,11 @@ def __repr__(self): -class MethodStandard(LookupBase): +class FieldStorageMethod(LookupBase): """ - Standard associated with the method. + Method of field storage. """ - __tablename__ = 'method_standard' + __tablename__ = 'field_storage_method' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1654,7 +1728,7 @@ class MethodStandard(LookupBase): def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1666,25 +1740,20 @@ def __repr__(self): -class Parameter(BaseEntity): +class HarvestMethod(LookupBase): """ - Parameter being measured. + Method of harvest. """ - __tablename__ = 'parameter' + __tablename__ = 'harvest_method' + id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + uri = Column(Text()) def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1696,11 +1765,11 @@ def __repr__(self): -class ParameterCategory(LookupBase): +class SoilType(LookupBase): """ - Category of parameter. + Type of soil. """ - __tablename__ = 'parameter_category' + __tablename__ = 'soil_type' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -1709,7 +1778,7 @@ class ParameterCategory(LookupBase): def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1721,18 +1790,14 @@ def __repr__(self): -class Experiment(BaseEntity): +class LocationSoilType(BaseEntity): """ - Experiment definition. + Soil type at a location. """ - __tablename__ = 'experiment' + __tablename__ = 'location_soil_type' - analyst_id = Column(Integer()) - exper_start_date = Column(Date()) - exper_duration = Column(Numeric()) - exper_duration_unit_id = Column(Integer()) - exper_location_id = Column(Integer()) - description = Column(Text()) + location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + soil_type_id = Column(Integer(), ForeignKey('SoilType.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1741,33 +1806,7 @@ class Experiment(BaseEntity): def __repr__(self): - return f"Experiment(analyst_id={self.analyst_id},exper_start_date={self.exper_start_date},exper_duration={self.exper_duration},exper_duration_unit_id={self.exper_duration_unit_id},exper_location_id={self.exper_location_id},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Equipment(LookupBase): - """ - Equipment used in experiments. - """ - __tablename__ = 'equipment' - - equipment_location_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Equipment(equipment_location_id={self.equipment_location_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1779,20 +1818,26 @@ def __repr__(self): -class AnalysisType(LookupBase): +class PhysicalCharacteristic(BaseEntity): """ - Type of analysis. + Physical characteristics of a sample. """ - __tablename__ = 'analysis_type' + __tablename__ = 'physical_characteristic' + field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) + particle_length = Column(Numeric()) + particle_width = Column(Numeric()) + particle_height = Column(Numeric()) + particle_unit_id = Column(Integer(), ForeignKey('Unit.id')) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1812,7 +1857,7 @@ class Dataset(BaseEntity): name = Column(Text()) record_type = Column(Text()) - source_id = Column(Integer()) + source_id = Column(Integer(), ForeignKey('DataSource.id')) start_date = Column(Date()) end_date = Column(Date()) description = Column(Text()) @@ -1867,15 +1912,15 @@ class Observation(BaseEntity): """ __tablename__ = 'observation' - dataset_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) record_type = Column(Text()) record_id = Column(Integer()) - parameter_id = Column(Integer()) + parameter_id = Column(Integer(), ForeignKey('Parameter.id')) value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) + unit_id = Column(Integer(), ForeignKey('Unit.id')) + dimension_type_id = Column(Integer(), ForeignKey('DimensionType.id')) dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) + dimension_unit_id = Column(Integer(), ForeignKey('Unit.id')) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -1897,59 +1942,19 @@ def __repr__(self): -class Aim1RecordBase(BaseEntity): - """ - - """ - __tablename__ = 'aim1_record_base' - - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_no = Column(Integer()) - technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Aim1RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Aim2RecordBase(BaseEntity): +class FacilityRecord(BaseEntity): """ - + Facility record. """ - __tablename__ = 'aim2_record_base' + __tablename__ = 'facility_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) - sample_id = Column(Integer()) - technical_replicate_total = Column(Integer()) - technical_replicate_no = Column(Integer()) - calc_method_id = Column(Integer()) - calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) - qc_pass = Column(Boolean()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + facility_name = Column(Text()) + location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + capacity_mw = Column(Numeric()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + operator = Column(Text()) + start_year = Column(Integer()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -1959,7 +1964,7 @@ class Aim2RecordBase(BaseEntity): def __repr__(self): - return f"Aim2RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"FacilityRecord(dataset_id={self.dataset_id},facility_name={self.facility_name},location_id={self.location_id},capacity_mw={self.capacity_mw},resource_id={self.resource_id},operator={self.operator},start_year={self.start_year},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -1971,13 +1976,12 @@ def __repr__(self): -class Strain(LookupBase): +class MethodAbbrev(LookupBase): """ - Strain used in fermentation. + Abbreviation for method. """ - __tablename__ = 'strain' + __tablename__ = 'method_abbrev' - parent_strain_id = Column(Integer()) id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) description = Column(Text()) @@ -1985,7 +1989,7 @@ class Strain(LookupBase): def __repr__(self): - return f"Strain(parent_strain_id={self.parent_strain_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -1997,11 +2001,11 @@ def __repr__(self): -class UsdaDomain(LookupBase): +class MethodCategory(LookupBase): """ - USDA domain. + Category of method. """ - __tablename__ = 'usda_domain' + __tablename__ = 'method_category' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -2010,7 +2014,7 @@ class UsdaDomain(LookupBase): def __repr__(self): - return f"UsdaDomain(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2022,11 +2026,11 @@ def __repr__(self): -class UsdaStatisticCategory(LookupBase): +class MethodStandard(LookupBase): """ - USDA statistic category. + Standard associated with the method. """ - __tablename__ = 'usda_statistic_category' + __tablename__ = 'method_standard' id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) @@ -2035,7 +2039,7 @@ class UsdaStatisticCategory(LookupBase): def __repr__(self): - return f"UsdaStatisticCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2047,20 +2051,25 @@ def __repr__(self): -class UsdaSurveyProgram(LookupBase): +class Parameter(BaseEntity): """ - USDA survey program. + Parameter being measured. """ - __tablename__ = 'usda_survey_program' + __tablename__ = 'parameter' - id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) + standard_unit_id = Column(Integer(), ForeignKey('Unit.id')) + calculated = Column(Boolean()) description = Column(Text()) - uri = Column(Text()) + id = Column(Integer(), primary_key=True, nullable=False ) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"UsdaSurveyProgram(id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2072,27 +2081,20 @@ def __repr__(self): -class UsdaCensusRecord(BaseEntity): +class ParameterCategory(LookupBase): """ - USDA census record. + Category of parameter. """ - __tablename__ = 'usda_census_record' + __tablename__ = 'parameter_category' - dataset_id = Column(Integer()) - geoid = Column(Text()) - commodity_code = Column(Integer()) - year = Column(Integer()) - source_reference = Column(Text()) - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"UsdaCensusRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},source_reference={self.source_reference},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2104,21 +2106,16 @@ def __repr__(self): -class UsdaSurveyRecord(BaseEntity): +class Contact(BaseEntity): """ - USDA survey record. + Contact information for a person. """ - __tablename__ = 'usda_survey_record' + __tablename__ = 'contact' - dataset_id = Column(Integer()) - geoid = Column(Text()) - commodity_code = Column(Integer()) - year = Column(Integer()) - survey_program_id = Column(Integer()) - survey_period = Column(Text()) - reference_month = Column(Text()) - seasonal_flag = Column(Boolean()) - note = Column(Text()) + first_name = Column(Text()) + last_name = Column(Text()) + email = Column(Text()) + affiliation = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -2127,7 +2124,7 @@ class UsdaSurveyRecord(BaseEntity): def __repr__(self): - return f"UsdaSurveyRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},survey_program_id={self.survey_program_id},survey_period={self.survey_period},reference_month={self.reference_month},seasonal_flag={self.seasonal_flag},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2139,23 +2136,22 @@ def __repr__(self): -class UsdaCommodity(LookupBase): +class Provider(BaseEntity): """ - USDA commodity. + Provider information. """ - __tablename__ = 'usda_commodity' + __tablename__ = 'provider' - usda_source = Column(Text()) - usda_code = Column(Text()) - parent_commodity_id = Column(Integer()) + codename = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) + created_at = Column(DateTime()) + updated_at = Column(DateTime()) + etl_run_id = Column(Text()) + lineage_group_id = Column(Integer()) def __repr__(self): - return f"UsdaCommodity(usda_source={self.usda_source},usda_code={self.usda_code},parent_commodity_id={self.parent_commodity_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" + return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2167,17 +2163,22 @@ def __repr__(self): -class UsdaTermMap(BaseEntity): +class Aim1RecordBase(BaseEntity): """ - Mapping of raw terms to USDA commodities. + """ - __tablename__ = 'usda_term_map' + __tablename__ = 'aim1_record_base' - source_system = Column(Text()) - source_context = Column(Text()) - raw_term = Column(Text()) - usda_commodity_id = Column(Integer()) - is_verified = Column(Boolean()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + sample_id = Column(Integer()) + technical_replicate_no = Column(Integer()) + technical_replicate_total = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -2187,7 +2188,7 @@ class UsdaTermMap(BaseEntity): def __repr__(self): - return f"UsdaTermMap(source_system={self.source_system},source_context={self.source_context},raw_term={self.raw_term},usda_commodity_id={self.usda_commodity_id},is_verified={self.is_verified},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Aim1RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2199,16 +2200,22 @@ def __repr__(self): -class ResourceUsdaCommodityMap(BaseEntity): +class Aim2RecordBase(BaseEntity): """ - Mapping between resources/crops and USDA commodities. + """ - __tablename__ = 'resource_usda_commodity_map' + __tablename__ = 'aim2_record_base' - resource_id = Column(Integer()) - primary_ag_product_id = Column(Integer()) - usda_commodity_id = Column(Integer()) - match_tier = Column(Text()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + sample_id = Column(Integer()) + technical_replicate_total = Column(Integer()) + technical_replicate_no = Column(Integer()) + calc_method_id = Column(Integer()) + calc_analyst_id = Column(Integer()) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -2218,7 +2225,7 @@ class ResourceUsdaCommodityMap(BaseEntity): def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Aim2RecordBase(dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2230,29 +2237,21 @@ def __repr__(self): -class UsdaMarketReport(BaseEntity): +class Strain(LookupBase): """ - USDA market report. + Strain used in fermentation. """ - __tablename__ = 'usda_market_report' + __tablename__ = 'strain' - slug_id = Column(Integer()) - slug_name = Column(Text()) - report_series_title = Column(Text()) - frequency = Column(Text()) - office_name = Column(Text()) - office_city_id = Column(Integer()) - office_state_fips = Column(Text()) - source_id = Column(Integer()) + parent_strain_id = Column(Integer()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"UsdaMarketReport(slug_id={self.slug_id},slug_name={self.slug_name},report_series_title={self.report_series_title},frequency={self.frequency},office_name={self.office_name},office_city_id={self.office_city_id},office_state_fips={self.office_state_fips},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"Strain(parent_strain_id={self.parent_strain_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2264,32 +2263,18 @@ def __repr__(self): -class UsdaMarketRecord(BaseEntity): +class ResourceAvailability(BaseEntity): """ - USDA market record. + Availability of a resource in a location. """ - __tablename__ = 'usda_market_record' + __tablename__ = 'resource_availability' - report_id = Column(Integer()) - dataset_id = Column(Integer()) - report_begin_date = Column(DateTime()) - report_end_date = Column(DateTime()) - report_date = Column(DateTime()) - commodity_id = Column(Integer()) - market_type_id = Column(Integer()) - market_type_category = Column(Text()) - grp = Column(Text()) - market_category_id = Column(Integer()) - class_ = Column(Text()) - grade = Column(Text()) - variety = Column(Text()) - protein_pct = Column(Numeric()) - application = Column(Text()) - pkg = Column(Text()) - sale_type = Column(Text()) - price_unit_id = Column(Integer()) - freight = Column(Text()) - trans_mode = Column(Text()) + resource_id = Column(Integer(), ForeignKey('Resource.id')) + geoid = Column(Text()) + from_month = Column(Integer()) + to_month = Column(Integer()) + year_round = Column(Boolean()) + note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -2298,7 +2283,7 @@ class UsdaMarketRecord(BaseEntity): def __repr__(self): - return f"UsdaMarketRecord(report_id={self.report_id},dataset_id={self.dataset_id},report_begin_date={self.report_begin_date},report_end_date={self.report_end_date},report_date={self.report_date},commodity_id={self.commodity_id},market_type_id={self.market_type_id},market_type_category={self.market_type_category},grp={self.grp},market_category_id={self.market_category_id},class_={self.class_},grade={self.grade},variety={self.variety},protein_pct={self.protein_pct},application={self.application},pkg={self.pkg},sale_type={self.sale_type},price_unit_id={self.price_unit_id},freight={self.freight},trans_mode={self.trans_mode},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2310,32 +2295,20 @@ def __repr__(self): -class LandiqRecord(BaseEntity): +class ResourceClass(LookupBase): """ - LandIQ record. + Classification of resources. """ - __tablename__ = 'landiq_record' + __tablename__ = 'resource_class' - dataset_id = Column(Integer()) - polygon_id = Column(Integer()) - main_crop = Column(Integer()) - secondary_crop = Column(Integer()) - tertiary_crop = Column(Integer()) - quaternary_crop = Column(Integer()) - confidence = Column(Integer()) - irrigated = Column(Boolean()) - acres = Column(Float()) - version = Column(Text()) - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"LandiqRecord(dataset_id={self.dataset_id},polygon_id={self.polygon_id},main_crop={self.main_crop},secondary_crop={self.secondary_crop},tertiary_crop={self.tertiary_crop},quaternary_crop={self.quaternary_crop},confidence={self.confidence},irrigated={self.irrigated},acres={self.acres},version={self.version},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2347,26 +2320,18 @@ def __repr__(self): -class BillionTon2023Record(BaseEntity): +class PreparationMethod(BaseEntity): """ - Billion Ton 2023 record. + Method of sample preparation. """ - __tablename__ = 'billion_ton2023_record' + __tablename__ = 'preparation_method' - subclass_id = Column(Integer()) - resource_id = Column(Integer()) - geoid = Column(Text()) - county_square_miles = Column(Float()) - model_name = Column(Text()) - scenario_name = Column(Text()) - price_offered_usd = Column(Numeric()) - production = Column(Integer()) - production_unit_id = Column(Integer()) - btu_ton = Column(Integer()) - production_energy_content = Column(Integer()) - energy_content_unit_id = Column(Integer()) - product_density_dtpersqmi = Column(Numeric()) - land_source = Column(Text()) + name = Column(Text()) + description = Column(Text()) + prep_method_abbrev_id = Column(Integer(), ForeignKey('PreparationMethodAbbreviation.id')) + prep_temp_c = Column(Numeric()) + uri = Column(Text()) + drying_step = Column(Boolean()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -2375,7 +2340,7 @@ class BillionTon2023Record(BaseEntity): def __repr__(self): - return f"BillionTon2023Record(subclass_id={self.subclass_id},resource_id={self.resource_id},geoid={self.geoid},county_square_miles={self.county_square_miles},model_name={self.model_name},scenario_name={self.scenario_name},price_offered_usd={self.price_offered_usd},production={self.production},production_unit_id={self.production_unit_id},btu_ton={self.btu_ton},production_energy_content={self.production_energy_content},energy_content_unit_id={self.energy_content_unit_id},product_density_dtpersqmi={self.product_density_dtpersqmi},land_source={self.land_source},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2387,29 +2352,20 @@ def __repr__(self): -class FacilityRecord(BaseEntity): +class PreparationMethodAbbreviation(LookupBase): """ - Facility record. + Abbreviation for preparation method. """ - __tablename__ = 'facility_record' + __tablename__ = 'preparation_method_abbreviation' - dataset_id = Column(Integer()) - facility_name = Column(Text()) - location_id = Column(Integer()) - capacity_mw = Column(Numeric()) - resource_id = Column(Integer()) - operator = Column(Text()) - start_year = Column(Integer()) - note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) + name = Column(Text()) + description = Column(Text()) + uri = Column(Text()) def __repr__(self): - return f"FacilityRecord(dataset_id={self.dataset_id},facility_name={self.facility_name},location_id={self.location_id},capacity_mw={self.capacity_mw},resource_id={self.resource_id},operator={self.operator},start_year={self.start_year},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" @@ -2427,15 +2383,15 @@ class ProximateRecord(Aim1RecordBase): """ __tablename__ = 'proximate_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2464,15 +2420,15 @@ class UltimateRecord(Aim1RecordBase): """ __tablename__ = 'ultimate_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2501,15 +2457,15 @@ class CompositionalRecord(Aim1RecordBase): """ __tablename__ = 'compositional_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2538,15 +2494,15 @@ class IcpRecord(Aim1RecordBase): """ __tablename__ = 'icp_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2575,19 +2531,19 @@ class XrfRecord(Aim1RecordBase): """ __tablename__ = 'xrf_record' - maybe_wavelength_nm = Column(Numeric()) - maybe_intensity = Column(Numeric()) - maybe_energy_slope = Column(Numeric()) - maybe_energy_offset = Column(Numeric()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + wavelength_nm = Column(Numeric()) + intensity = Column(Numeric()) + energy_slope = Column(Numeric()) + energy_offset = Column(Numeric()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2598,7 +2554,7 @@ class XrfRecord(Aim1RecordBase): def __repr__(self): - return f"XrfRecord(maybe_wavelength_nm={self.maybe_wavelength_nm},maybe_intensity={self.maybe_intensity},maybe_energy_slope={self.maybe_energy_slope},maybe_energy_offset={self.maybe_energy_offset},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"XrfRecord(wavelength_nm={self.wavelength_nm},intensity={self.intensity},energy_slope={self.energy_slope},energy_offset={self.energy_offset},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2616,17 +2572,17 @@ class XrdRecord(Aim1RecordBase): """ __tablename__ = 'xrd_record' - maybe_scan_low_nm = Column(Integer()) - maybe_scan_high_nm = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + scan_low_nm = Column(Integer()) + scan_high_nm = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2637,7 +2593,7 @@ class XrdRecord(Aim1RecordBase): def __repr__(self): - return f"XrdRecord(maybe_scan_low_nm={self.maybe_scan_low_nm},maybe_scan_high_nm={self.maybe_scan_high_nm},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"XrdRecord(scan_low_nm={self.scan_low_nm},scan_high_nm={self.scan_high_nm},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_no={self.technical_replicate_no},technical_replicate_total={self.technical_replicate_total},method_id={self.method_id},analyst_id={self.analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2655,15 +2611,15 @@ class CalorimetryRecord(Aim1RecordBase): """ __tablename__ = 'calorimetry_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2692,15 +2648,15 @@ class FtnirRecord(Aim1RecordBase): """ __tablename__ = 'ftnir_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2729,15 +2685,15 @@ class RgbRecord(Aim1RecordBase): """ __tablename__ = 'rgb_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer()) - analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + method_id = Column(Integer(), ForeignKey('Method.id')) + analyst_id = Column(Integer(), ForeignKey('Contact.id')) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2772,15 +2728,15 @@ class PretreatmentRecord(Aim2RecordBase): block_position = Column(Text()) temperature = Column(Numeric()) replicate_no = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2812,21 +2768,20 @@ class FermentationRecord(Aim2RecordBase): strain_id = Column(Integer()) pretreatment_method_id = Column(Integer()) eh_method_id = Column(Integer()) - replicate_no = Column(Integer()) well_position = Column(Text()) temperature = Column(Numeric()) agitation_rpm = Column(Numeric()) vessel_id = Column(Integer()) analyte_detection_equipment_id = Column(Integer()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2837,7 +2792,7 @@ class FermentationRecord(Aim2RecordBase): def __repr__(self): - return f"FermentationRecord(strain_id={self.strain_id},pretreatment_method_id={self.pretreatment_method_id},eh_method_id={self.eh_method_id},replicate_no={self.replicate_no},well_position={self.well_position},temperature={self.temperature},agitation_rpm={self.agitation_rpm},vessel_id={self.vessel_id},analyte_detection_equipment_id={self.analyte_detection_equipment_id},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" + return f"FermentationRecord(strain_id={self.strain_id},pretreatment_method_id={self.pretreatment_method_id},eh_method_id={self.eh_method_id},well_position={self.well_position},temperature={self.temperature},agitation_rpm={self.agitation_rpm},vessel_id={self.vessel_id},analyte_detection_equipment_id={self.analyte_detection_equipment_id},dataset_id={self.dataset_id},experiment_id={self.experiment_id},resource_id={self.resource_id},sample_id={self.sample_id},technical_replicate_total={self.technical_replicate_total},technical_replicate_no={self.technical_replicate_no},calc_method_id={self.calc_method_id},calc_analyst_id={self.calc_analyst_id},raw_data_id={self.raw_data_id},qc_pass={self.qc_pass},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" @@ -2858,15 +2813,15 @@ class GasificationRecord(Aim2RecordBase): feedstock_mass = Column(Numeric()) bed_temperature = Column(Numeric()) gas_flow_rate = Column(Numeric()) - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2895,15 +2850,15 @@ class AutoclaveRecord(Aim2RecordBase): """ __tablename__ = 'autoclave_record' - dataset_id = Column(Integer()) - experiment_id = Column(Integer()) - resource_id = Column(Integer()) + dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + experiment_id = Column(Integer(), ForeignKey('Experiment.id')) + resource_id = Column(Integer(), ForeignKey('Resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer()) + raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/core.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/core.py deleted file mode 100644 index 20ba55a2..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/core.py +++ /dev/null @@ -1,46 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py deleted file mode 100644 index 10206eab..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/data_sources_metadata.py +++ /dev/null @@ -1,1030 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py deleted file mode 100644 index d27ecc16..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/experiment_equipment.py +++ /dev/null @@ -1,1371 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class ExperimentMethod(Base): - """ - Link between Experiment and Method. - """ - __tablename__ = 'experiment_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - method_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentMethod(id={self.id},experiment_id={self.experiment_id},method_id={self.method_id},)" - - - - - - -class ExperimentEquipment(Base): - """ - Link between Experiment and Equipment. - """ - __tablename__ = 'experiment_equipment' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - equipment_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentEquipment(id={self.id},experiment_id={self.experiment_id},equipment_id={self.equipment_id},)" - - - - - - -class ExperimentAnalysis(Base): - """ - Link between Experiment and AnalysisType. - """ - __tablename__ = 'experiment_analysis' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - analysis_type_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentAnalysis(id={self.id},experiment_id={self.experiment_id},analysis_type_id={self.analysis_type_id},)" - - - - - - -class ExperimentPreparedSample(Base): - """ - Link between Experiment and PreparedSample. - """ - __tablename__ = 'experiment_prepared_sample' - - id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer()) - prepared_sample_id = Column(Integer()) - - - def __repr__(self): - return f"ExperimentPreparedSample(id={self.id},experiment_id={self.experiment_id},prepared_sample_id={self.prepared_sample_id},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class Experiment(BaseEntity): - """ - Experiment definition. - """ - __tablename__ = 'experiment' - - analyst_id = Column(Integer()) - exper_start_date = Column(Date()) - exper_duration = Column(Numeric()) - exper_duration_unit_id = Column(Integer()) - exper_location_id = Column(Integer()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Experiment(analyst_id={self.analyst_id},exper_start_date={self.exper_start_date},exper_duration={self.exper_duration},exper_duration_unit_id={self.exper_duration_unit_id},exper_location_id={self.exper_location_id},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Equipment(LookupBase): - """ - Equipment used in experiments. - """ - __tablename__ = 'equipment' - - equipment_location_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Equipment(equipment_location_id={self.equipment_location_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethod(BaseEntity): - """ - Method of sample preparation. - """ - __tablename__ = 'preparation_method' - - name = Column(Text()) - description = Column(Text()) - prep_method_abbrev_id = Column(Integer()) - prep_temp_c = Column(Numeric()) - uri = Column(Text()) - drying_step = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethodAbbreviation(LookupBase): - """ - Abbreviation for preparation method. - """ - __tablename__ = 'preparation_method_abbreviation' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparedSample(BaseEntity): - """ - Sample that has been prepared. - """ - __tablename__ = 'prepared_sample' - - name = Column(Text()) - field_sample_id = Column(Integer()) - prep_method_id = Column(Integer()) - prep_date = Column(Date()) - preparer_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py deleted file mode 100644 index deaaf7e6..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/external_data.py +++ /dev/null @@ -1,1557 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class Polygon(Base): - """ - Geospatial polygon. - """ - __tablename__ = 'polygon' - - id = Column(Integer(), primary_key=True, nullable=False ) - geoid = Column(Text()) - geom = Column(Text()) - - - def __repr__(self): - return f"Polygon(id={self.id},geoid={self.geoid},geom={self.geom},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class UsdaDomain(LookupBase): - """ - USDA domain. - """ - __tablename__ = 'usda_domain' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"UsdaDomain(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaStatisticCategory(LookupBase): - """ - USDA statistic category. - """ - __tablename__ = 'usda_statistic_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"UsdaStatisticCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaSurveyProgram(LookupBase): - """ - USDA survey program. - """ - __tablename__ = 'usda_survey_program' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"UsdaSurveyProgram(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaCensusRecord(BaseEntity): - """ - USDA census record. - """ - __tablename__ = 'usda_census_record' - - dataset_id = Column(Integer()) - geoid = Column(Text()) - commodity_code = Column(Integer()) - year = Column(Integer()) - source_reference = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UsdaCensusRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},source_reference={self.source_reference},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaSurveyRecord(BaseEntity): - """ - USDA survey record. - """ - __tablename__ = 'usda_survey_record' - - dataset_id = Column(Integer()) - geoid = Column(Text()) - commodity_code = Column(Integer()) - year = Column(Integer()) - survey_program_id = Column(Integer()) - survey_period = Column(Text()) - reference_month = Column(Text()) - seasonal_flag = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UsdaSurveyRecord(dataset_id={self.dataset_id},geoid={self.geoid},commodity_code={self.commodity_code},year={self.year},survey_program_id={self.survey_program_id},survey_period={self.survey_period},reference_month={self.reference_month},seasonal_flag={self.seasonal_flag},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaCommodity(LookupBase): - """ - USDA commodity. - """ - __tablename__ = 'usda_commodity' - - usda_source = Column(Text()) - usda_code = Column(Text()) - parent_commodity_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"UsdaCommodity(usda_source={self.usda_source},usda_code={self.usda_code},parent_commodity_id={self.parent_commodity_id},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaTermMap(BaseEntity): - """ - Mapping of raw terms to USDA commodities. - """ - __tablename__ = 'usda_term_map' - - source_system = Column(Text()) - source_context = Column(Text()) - raw_term = Column(Text()) - usda_commodity_id = Column(Integer()) - is_verified = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UsdaTermMap(source_system={self.source_system},source_context={self.source_context},raw_term={self.raw_term},usda_commodity_id={self.usda_commodity_id},is_verified={self.is_verified},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceUsdaCommodityMap(BaseEntity): - """ - Mapping between resources/crops and USDA commodities. - """ - __tablename__ = 'resource_usda_commodity_map' - - resource_id = Column(Integer()) - primary_ag_product_id = Column(Integer()) - usda_commodity_id = Column(Integer()) - match_tier = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceUsdaCommodityMap(resource_id={self.resource_id},primary_ag_product_id={self.primary_ag_product_id},usda_commodity_id={self.usda_commodity_id},match_tier={self.match_tier},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaMarketReport(BaseEntity): - """ - USDA market report. - """ - __tablename__ = 'usda_market_report' - - slug_id = Column(Integer()) - slug_name = Column(Text()) - report_series_title = Column(Text()) - frequency = Column(Text()) - office_name = Column(Text()) - office_city_id = Column(Integer()) - office_state_fips = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UsdaMarketReport(slug_id={self.slug_id},slug_name={self.slug_name},report_series_title={self.report_series_title},frequency={self.frequency},office_name={self.office_name},office_city_id={self.office_city_id},office_state_fips={self.office_state_fips},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class UsdaMarketRecord(BaseEntity): - """ - USDA market record. - """ - __tablename__ = 'usda_market_record' - - report_id = Column(Integer()) - dataset_id = Column(Integer()) - report_begin_date = Column(DateTime()) - report_end_date = Column(DateTime()) - report_date = Column(DateTime()) - commodity_id = Column(Integer()) - market_type_id = Column(Integer()) - market_type_category = Column(Text()) - grp = Column(Text()) - market_category_id = Column(Integer()) - class_ = Column(Text()) - grade = Column(Text()) - variety = Column(Text()) - protein_pct = Column(Numeric()) - application = Column(Text()) - pkg = Column(Text()) - sale_type = Column(Text()) - price_unit_id = Column(Integer()) - freight = Column(Text()) - trans_mode = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"UsdaMarketRecord(report_id={self.report_id},dataset_id={self.dataset_id},report_begin_date={self.report_begin_date},report_end_date={self.report_end_date},report_date={self.report_date},commodity_id={self.commodity_id},market_type_id={self.market_type_id},market_type_category={self.market_type_category},grp={self.grp},market_category_id={self.market_category_id},class_={self.class_},grade={self.grade},variety={self.variety},protein_pct={self.protein_pct},application={self.application},pkg={self.pkg},sale_type={self.sale_type},price_unit_id={self.price_unit_id},freight={self.freight},trans_mode={self.trans_mode},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LandiqRecord(BaseEntity): - """ - LandIQ record. - """ - __tablename__ = 'landiq_record' - - dataset_id = Column(Integer()) - polygon_id = Column(Integer()) - main_crop = Column(Integer()) - secondary_crop = Column(Integer()) - tertiary_crop = Column(Integer()) - quaternary_crop = Column(Integer()) - confidence = Column(Integer()) - irrigated = Column(Boolean()) - acres = Column(Float()) - version = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LandiqRecord(dataset_id={self.dataset_id},polygon_id={self.polygon_id},main_crop={self.main_crop},secondary_crop={self.secondary_crop},tertiary_crop={self.tertiary_crop},quaternary_crop={self.quaternary_crop},confidence={self.confidence},irrigated={self.irrigated},acres={self.acres},version={self.version},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class BillionTon2023Record(BaseEntity): - """ - Billion Ton 2023 record. - """ - __tablename__ = 'billion_ton2023_record' - - subclass_id = Column(Integer()) - resource_id = Column(Integer()) - geoid = Column(Text()) - county_square_miles = Column(Float()) - model_name = Column(Text()) - scenario_name = Column(Text()) - price_offered_usd = Column(Numeric()) - production = Column(Integer()) - production_unit_id = Column(Integer()) - btu_ton = Column(Integer()) - production_energy_content = Column(Integer()) - energy_content_unit_id = Column(Integer()) - product_density_dtpersqmi = Column(Numeric()) - land_source = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BillionTon2023Record(subclass_id={self.subclass_id},resource_id={self.resource_id},geoid={self.geoid},county_square_miles={self.county_square_miles},model_name={self.model_name},scenario_name={self.scenario_name},price_offered_usd={self.price_offered_usd},production={self.production},production_unit_id={self.production_unit_id},btu_ton={self.btu_ton},production_energy_content={self.production_energy_content},energy_content_unit_id={self.energy_content_unit_id},product_density_dtpersqmi={self.product_density_dtpersqmi},land_source={self.land_source},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py deleted file mode 100644 index 0e5c3b47..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/field_sampling.py +++ /dev/null @@ -1,1030 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py deleted file mode 100644 index 5e8fd492..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/general_analysis.py +++ /dev/null @@ -1,1148 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/geography.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/geography.py deleted file mode 100644 index 349885b9..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/geography.py +++ /dev/null @@ -1,106 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class Geography(Base): - """ - Geographic region definition (e.g. county, state). - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class LocationAddress(BaseEntity): - """ - Specific physical location. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py deleted file mode 100644 index 021a9423..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/infrastructure.py +++ /dev/null @@ -1,1625 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class InfrastructureBiodieselPlants(Base): - """ - Biodiesel plants infrastructure. - """ - __tablename__ = 'infrastructure_biodiesel_plants' - - biodiesel_plant_id = Column(Integer(), primary_key=True, nullable=False ) - company = Column(Text()) - bbi_index = Column(Integer()) - city = Column(Text()) - state = Column(Text()) - capacity_mmg_per_y = Column(Integer()) - feedstock = Column(Text()) - status = Column(Text()) - address = Column(Text()) - coordinates = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - source = Column(Text()) - - - def __repr__(self): - return f"InfrastructureBiodieselPlants(biodiesel_plant_id={self.biodiesel_plant_id},company={self.company},bbi_index={self.bbi_index},city={self.city},state={self.state},capacity_mmg_per_y={self.capacity_mmg_per_y},feedstock={self.feedstock},status={self.status},address={self.address},coordinates={self.coordinates},latitude={self.latitude},longitude={self.longitude},source={self.source},)" - - - - - - -class InfrastructureBiosolidsFacilities(Base): - """ - Biosolids facilities infrastructure. - """ - __tablename__ = 'infrastructure_biosolids_facilities' - - biosolid_facility_id = Column(Integer(), primary_key=True, nullable=False ) - report_submitted_date = Column(Date()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - facility = Column(Text()) - authority = Column(Text()) - plant_type = Column(Text()) - aqmd = Column(Text()) - facility_address = Column(Text()) - facility_city = Column(Text()) - state = Column(Text()) - facility_zip = Column(Text()) - facility_county = Column(Text()) - mailing_street_1 = Column(Text()) - mailing_city = Column(Text()) - mailing_state = Column(Text()) - mailing_zip = Column(Text()) - biosolids_number = Column(Text()) - biosolids_contact = Column(Text()) - biosolids_contact_phone = Column(Text()) - biosolids_contact_email = Column(Text()) - adwf = Column(Numeric()) - potw_biosolids_generated = Column(Integer()) - twtds_biosolids_treated = Column(Integer()) - class_b_land_app = Column(Integer()) - class_b_applier = Column(Text()) - class_a_compost = Column(Integer()) - class_a_heat_dried = Column(Integer()) - class_a_other = Column(Integer()) - class_a_other_applier = Column(Text()) - twtds_transfer_to_second_preparer = Column(Integer()) - twtds_second_preparer_name = Column(Text()) - adc_or_final_c = Column(Integer()) - landfill = Column(Integer()) - landfill_name = Column(Text()) - surface_disposal = Column(Integer()) - deepwell_injection = Column(Text()) - stored = Column(Integer()) - longterm_treatment = Column(Integer()) - other = Column(Integer()) - name_of_other = Column(Text()) - incineration = Column(Integer()) - - - def __repr__(self): - return f"InfrastructureBiosolidsFacilities(biosolid_facility_id={self.biosolid_facility_id},report_submitted_date={self.report_submitted_date},latitude={self.latitude},longitude={self.longitude},facility={self.facility},authority={self.authority},plant_type={self.plant_type},aqmd={self.aqmd},facility_address={self.facility_address},facility_city={self.facility_city},state={self.state},facility_zip={self.facility_zip},facility_county={self.facility_county},mailing_street_1={self.mailing_street_1},mailing_city={self.mailing_city},mailing_state={self.mailing_state},mailing_zip={self.mailing_zip},biosolids_number={self.biosolids_number},biosolids_contact={self.biosolids_contact},biosolids_contact_phone={self.biosolids_contact_phone},biosolids_contact_email={self.biosolids_contact_email},adwf={self.adwf},potw_biosolids_generated={self.potw_biosolids_generated},twtds_biosolids_treated={self.twtds_biosolids_treated},class_b_land_app={self.class_b_land_app},class_b_applier={self.class_b_applier},class_a_compost={self.class_a_compost},class_a_heat_dried={self.class_a_heat_dried},class_a_other={self.class_a_other},class_a_other_applier={self.class_a_other_applier},twtds_transfer_to_second_preparer={self.twtds_transfer_to_second_preparer},twtds_second_preparer_name={self.twtds_second_preparer_name},adc_or_final_c={self.adc_or_final_c},landfill={self.landfill},landfill_name={self.landfill_name},surface_disposal={self.surface_disposal},deepwell_injection={self.deepwell_injection},stored={self.stored},longterm_treatment={self.longterm_treatment},other={self.other},name_of_other={self.name_of_other},incineration={self.incineration},)" - - - - - - -class InfrastructureCafoManureLocations(Base): - """ - CAFO manure locations infrastructure. - """ - __tablename__ = 'infrastructure_cafo_manure_locations' - - cafo_manure_id = Column(Integer(), primary_key=True, nullable=False ) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - owner_name = Column(Text()) - facility_name = Column(Text()) - address = Column(Text()) - town = Column(Text()) - state = Column(Text()) - zip = Column(Text()) - animal = Column(Text()) - animal_feed_operation_type = Column(Text()) - animal_units = Column(Integer()) - animal_count = Column(Integer()) - manure_total_solids = Column(Numeric()) - source = Column(Text()) - date_accessed = Column(Date()) - - - def __repr__(self): - return f"InfrastructureCafoManureLocations(cafo_manure_id={self.cafo_manure_id},latitude={self.latitude},longitude={self.longitude},owner_name={self.owner_name},facility_name={self.facility_name},address={self.address},town={self.town},state={self.state},zip={self.zip},animal={self.animal},animal_feed_operation_type={self.animal_feed_operation_type},animal_units={self.animal_units},animal_count={self.animal_count},manure_total_solids={self.manure_total_solids},source={self.source},date_accessed={self.date_accessed},)" - - - - - - -class InfrastructureEthanolBiorefineries(Base): - """ - Ethanol biorefineries infrastructure. - """ - __tablename__ = 'infrastructure_ethanol_biorefineries' - - ethanol_biorefinery_id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - city = Column(Text()) - state = Column(Text()) - address = Column(Text()) - type = Column(Text()) - capacity_mgy = Column(Integer()) - production_mgy = Column(Integer()) - constr_exp = Column(Integer()) - - - def __repr__(self): - return f"InfrastructureEthanolBiorefineries(ethanol_biorefinery_id={self.ethanol_biorefinery_id},name={self.name},city={self.city},state={self.state},address={self.address},type={self.type},capacity_mgy={self.capacity_mgy},production_mgy={self.production_mgy},constr_exp={self.constr_exp},)" - - - - - - -class InfrastructureLandfills(Base): - """ - Landfills infrastructure. - """ - __tablename__ = 'infrastructure_landfills' - - project_id = Column(Text(), primary_key=True, nullable=False ) - project_int_id = Column(Integer()) - ghgrp_id = Column(Text()) - landfill_id = Column(Integer()) - landfill_name = Column(Text()) - state = Column(Text()) - physical_address = Column(Text()) - city = Column(Text()) - county = Column(Text()) - zip_code = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - ownership_type = Column(Text()) - landfill_owner_orgs = Column(Text()) - landfill_opened_year = Column(Date()) - landfill_closure_year = Column(Date()) - landfill_status = Column(Text()) - waste_in_place = Column(Integer()) - waste_in_place_year = Column(Date()) - lfg_system_in_place = Column(Boolean()) - lfg_collected = Column(Numeric()) - lfg_flared = Column(Numeric()) - project_status = Column(Text()) - project_name = Column(Text()) - project_start_date = Column(Date()) - project_shutdown_date = Column(Date()) - project_type_category = Column(Text()) - lfg_energy_project_type = Column(Text()) - rng_delivery_method = Column(Text()) - actual_mw_generation = Column(Numeric()) - rated_mw_capacity = Column(Numeric()) - lfg_flow_to_project = Column(Numeric()) - direct_emission_reductions = Column(Numeric()) - avoided_emission_reductions = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureLandfills(project_id={self.project_id},project_int_id={self.project_int_id},ghgrp_id={self.ghgrp_id},landfill_id={self.landfill_id},landfill_name={self.landfill_name},state={self.state},physical_address={self.physical_address},city={self.city},county={self.county},zip_code={self.zip_code},latitude={self.latitude},longitude={self.longitude},ownership_type={self.ownership_type},landfill_owner_orgs={self.landfill_owner_orgs},landfill_opened_year={self.landfill_opened_year},landfill_closure_year={self.landfill_closure_year},landfill_status={self.landfill_status},waste_in_place={self.waste_in_place},waste_in_place_year={self.waste_in_place_year},lfg_system_in_place={self.lfg_system_in_place},lfg_collected={self.lfg_collected},lfg_flared={self.lfg_flared},project_status={self.project_status},project_name={self.project_name},project_start_date={self.project_start_date},project_shutdown_date={self.project_shutdown_date},project_type_category={self.project_type_category},lfg_energy_project_type={self.lfg_energy_project_type},rng_delivery_method={self.rng_delivery_method},actual_mw_generation={self.actual_mw_generation},rated_mw_capacity={self.rated_mw_capacity},lfg_flow_to_project={self.lfg_flow_to_project},direct_emission_reductions={self.direct_emission_reductions},avoided_emission_reductions={self.avoided_emission_reductions},)" - - - - - - -class InfrastructureLivestockAnaerobicDigesters(Base): - """ - Livestock anaerobic digesters infrastructure. - """ - __tablename__ = 'infrastructure_livestock_anaerobic_digesters' - - digester_id = Column(Integer(), primary_key=True, nullable=False ) - project_name = Column(Text()) - project_type = Column(Text()) - city = Column(Text()) - state = Column(Text()) - digester_type = Column(Text()) - profile = Column(Text()) - year_operational = Column(Date()) - animal_type_class = Column(Text()) - animal_types = Column(Text()) - pop_feeding_digester = Column(Text()) - total_pop_feeding_digester = Column(Integer()) - cattle = Column(Integer()) - dairy = Column(Integer()) - poultry = Column(Integer()) - swine = Column(Integer()) - codigestion = Column(Text()) - biogas_generation_estimate = Column(Integer()) - electricity_generated = Column(Integer()) - biogas_end_uses = Column(Text()) - methane_emission_reductions = Column(Integer()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureLivestockAnaerobicDigesters(digester_id={self.digester_id},project_name={self.project_name},project_type={self.project_type},city={self.city},state={self.state},digester_type={self.digester_type},profile={self.profile},year_operational={self.year_operational},animal_type_class={self.animal_type_class},animal_types={self.animal_types},pop_feeding_digester={self.pop_feeding_digester},total_pop_feeding_digester={self.total_pop_feeding_digester},cattle={self.cattle},dairy={self.dairy},poultry={self.poultry},swine={self.swine},codigestion={self.codigestion},biogas_generation_estimate={self.biogas_generation_estimate},electricity_generated={self.electricity_generated},biogas_end_uses={self.biogas_end_uses},methane_emission_reductions={self.methane_emission_reductions},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class InfrastructureSafAndRenewableDieselPlants(Base): - """ - SAF and renewable diesel plants infrastructure. - """ - __tablename__ = 'infrastructure_saf_and_renewable_diesel_plants' - - ibcc_index = Column(Integer(), primary_key=True, nullable=False ) - company = Column(Text()) - city = Column(Text()) - state = Column(Text()) - country = Column(Text()) - capacity = Column(Text()) - feedstock = Column(Text()) - products = Column(Text()) - status = Column(Text()) - address = Column(Text()) - coordinates = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureSafAndRenewableDieselPlants(ibcc_index={self.ibcc_index},company={self.company},city={self.city},state={self.state},country={self.country},capacity={self.capacity},feedstock={self.feedstock},products={self.products},status={self.status},address={self.address},coordinates={self.coordinates},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class InfrastructureWastewaterTreatmentPlants(Base): - """ - Wastewater treatment plants infrastructure. - """ - __tablename__ = 'infrastructure_wastewater_treatment_plants' - - plant_id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - state = Column(Text()) - codigestion = Column(Text()) - flow_design_adjusted = Column(Numeric()) - flow_average = Column(Numeric()) - biosolids = Column(Numeric()) - excess_flow = Column(Numeric()) - biogas_utilized = Column(Boolean()) - flaring = Column(Boolean()) - excess_mass_loading_rate = Column(Numeric()) - excess_mass_loading_rate_wet = Column(Numeric()) - methane_production = Column(Numeric()) - energy_content = Column(Numeric()) - electric_kw = Column(Numeric()) - thermal_mmbtu_d = Column(Numeric()) - electric_kwh = Column(Numeric()) - thermal_annual_mmbtu_y = Column(Numeric()) - anaerobic_digestion_facility = Column(Text()) - county = Column(Text()) - dayload_bdt = Column(Numeric()) - dayload = Column(Numeric()) - equivalent_generation = Column(Numeric()) - facility_type = Column(Text()) - feedstock = Column(Text()) - type = Column(Text()) - city = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - zipcode = Column(Text()) - - - def __repr__(self): - return f"InfrastructureWastewaterTreatmentPlants(plant_id={self.plant_id},name={self.name},state={self.state},codigestion={self.codigestion},flow_design_adjusted={self.flow_design_adjusted},flow_average={self.flow_average},biosolids={self.biosolids},excess_flow={self.excess_flow},biogas_utilized={self.biogas_utilized},flaring={self.flaring},excess_mass_loading_rate={self.excess_mass_loading_rate},excess_mass_loading_rate_wet={self.excess_mass_loading_rate_wet},methane_production={self.methane_production},energy_content={self.energy_content},electric_kw={self.electric_kw},thermal_mmbtu_d={self.thermal_mmbtu_d},electric_kwh={self.electric_kwh},thermal_annual_mmbtu_y={self.thermal_annual_mmbtu_y},anaerobic_digestion_facility={self.anaerobic_digestion_facility},county={self.county},dayload_bdt={self.dayload_bdt},dayload={self.dayload},equivalent_generation={self.equivalent_generation},facility_type={self.facility_type},feedstock={self.feedstock},type={self.type},city={self.city},latitude={self.latitude},longitude={self.longitude},zipcode={self.zipcode},)" - - - - - - -class InfrastructureCombustionPlants(Base): - """ - Combustion plants infrastructure. - """ - __tablename__ = 'infrastructure_combustion_plants' - - combustion_fid = Column(Integer(), primary_key=True, nullable=False ) - objectid = Column(Integer()) - status = Column(Text()) - city = Column(Text()) - name = Column(Text()) - county = Column(Text()) - equivalent_generation = Column(Numeric()) - np_mw = Column(Numeric()) - cf = Column(Numeric()) - yearload = Column(Integer()) - fuel = Column(Text()) - notes = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureCombustionPlants(combustion_fid={self.combustion_fid},objectid={self.objectid},status={self.status},city={self.city},name={self.name},county={self.county},equivalent_generation={self.equivalent_generation},np_mw={self.np_mw},cf={self.cf},yearload={self.yearload},fuel={self.fuel},notes={self.notes},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class InfrastructureDistrictEnergySystems(Base): - """ - District energy systems infrastructure. - """ - __tablename__ = 'infrastructure_district_energy_systems' - - des_fid = Column(Integer(), primary_key=True, nullable=False ) - cbg_id = Column(Integer()) - name = Column(Text()) - system = Column(Text()) - object_id = Column(Integer()) - city = Column(Text()) - state = Column(Text()) - primary_fuel = Column(Text()) - secondary_fuel = Column(Text()) - usetype = Column(Text()) - cap_st = Column(Numeric()) - cap_hw = Column(Numeric()) - cap_cw = Column(Numeric()) - chpcg_cap = Column(Numeric()) - excess_c = Column(Numeric()) - excess_h = Column(Numeric()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureDistrictEnergySystems(des_fid={self.des_fid},cbg_id={self.cbg_id},name={self.name},system={self.system},object_id={self.object_id},city={self.city},state={self.state},primary_fuel={self.primary_fuel},secondary_fuel={self.secondary_fuel},usetype={self.usetype},cap_st={self.cap_st},cap_hw={self.cap_hw},cap_cw={self.cap_cw},chpcg_cap={self.chpcg_cap},excess_c={self.excess_c},excess_h={self.excess_h},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class InfrastructureFoodProcessingFacilities(Base): - """ - Food processing facilities infrastructure. - """ - __tablename__ = 'infrastructure_food_processing_facilities' - - processing_facility_id = Column(Integer(), primary_key=True, nullable=False ) - address = Column(Text()) - county = Column(Text()) - city = Column(Text()) - company = Column(Text()) - join_count = Column(Integer()) - master_type = Column(Text()) - state = Column(Text()) - subtype = Column(Text()) - target_fid = Column(Integer()) - processing_type = Column(Text()) - zip = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureFoodProcessingFacilities(processing_facility_id={self.processing_facility_id},address={self.address},county={self.county},city={self.city},company={self.company},join_count={self.join_count},master_type={self.master_type},state={self.state},subtype={self.subtype},target_fid={self.target_fid},processing_type={self.processing_type},zip={self.zip},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class InfrastructureMswToEnergyAnaerobicDigesters(Base): - """ - MSW to energy anaerobic digesters infrastructure. - """ - __tablename__ = 'infrastructure_msw_to_energy_anaerobic_digesters' - - wte_id = Column(Integer(), primary_key=True, nullable=False ) - city = Column(Text()) - county = Column(Text()) - equivalent_generation = Column(Numeric()) - feedstock = Column(Text()) - dayload = Column(Numeric()) - dayload_bdt = Column(Numeric()) - facility_type = Column(Text()) - status = Column(Text()) - notes = Column(Text()) - source = Column(Text()) - type = Column(Text()) - wkt_geom = Column(Text()) - geom = Column(Text()) - latitude = Column(Numeric()) - longitude = Column(Numeric()) - - - def __repr__(self): - return f"InfrastructureMswToEnergyAnaerobicDigesters(wte_id={self.wte_id},city={self.city},county={self.county},equivalent_generation={self.equivalent_generation},feedstock={self.feedstock},dayload={self.dayload},dayload_bdt={self.dayload_bdt},facility_type={self.facility_type},status={self.status},notes={self.notes},source={self.source},type={self.type},wkt_geom={self.wkt_geom},geom={self.geom},latitude={self.latitude},longitude={self.longitude},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class FacilityRecord(BaseEntity): - """ - Facility record. - """ - __tablename__ = 'facility_record' - - dataset_id = Column(Integer()) - facility_name = Column(Text()) - location_id = Column(Integer()) - capacity_mw = Column(Numeric()) - resource_id = Column(Integer()) - operator = Column(Text()) - start_year = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FacilityRecord(dataset_id={self.dataset_id},facility_name={self.facility_name},location_id={self.location_id},capacity_mw={self.capacity_mw},resource_id={self.resource_id},operator={self.operator},start_year={self.start_year},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AnalysisType(LookupBase): - """ - Type of analysis. - """ - __tablename__ = 'analysis_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AnalysisType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Dataset(BaseEntity): - """ - Dataset definition. - """ - __tablename__ = 'dataset' - - name = Column(Text()) - record_type = Column(Text()) - source_id = Column(Integer()) - start_date = Column(Date()) - end_date = Column(Date()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Dataset(name={self.name},record_type={self.record_type},source_id={self.source_id},start_date={self.start_date},end_date={self.end_date},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DimensionType(LookupBase): - """ - Type of dimension. - """ - __tablename__ = 'dimension_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"DimensionType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Observation(BaseEntity): - """ - Observation data. - """ - __tablename__ = 'observation' - - dataset_id = Column(Integer()) - record_type = Column(Text()) - record_id = Column(Integer()) - parameter_id = Column(Integer()) - value = Column(Numeric()) - unit_id = Column(Integer()) - dimension_type_id = Column(Integer()) - dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Observation(dataset_id={self.dataset_id},record_type={self.record_type},record_id={self.record_id},parameter_id={self.parameter_id},value={self.value},unit_id={self.unit_id},dimension_type_id={self.dimension_type_id},dimension_value={self.dimension_value},dimension_unit_id={self.dimension_unit_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/lineage.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/lineage.py deleted file mode 100644 index c1fb6e9a..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/lineage.py +++ /dev/null @@ -1,109 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class LineageGroup(Base): - """ - Grouping for lineage information. - """ - __tablename__ = 'lineage_group' - - id = Column(Integer(), primary_key=True, nullable=False ) - etl_run_id = Column(Text()) - note = Column(Text()) - - - def __repr__(self): - return f"LineageGroup(id={self.id},etl_run_id={self.etl_run_id},note={self.note},)" - - - - - - -class EntityLineage(Base): - """ - Lineage information for a specific entity. - """ - __tablename__ = 'entity_lineage' - - id = Column(Integer(), primary_key=True, nullable=False ) - lineage_group_id = Column(Integer()) - source_table = Column(Text()) - source_row_id = Column(Text()) - note = Column(Text()) - - - def __repr__(self): - return f"EntityLineage(id={self.id},lineage_group_id={self.lineage_group_id},source_table={self.source_table},source_row_id={self.source_row_id},note={self.note},)" - - - - - - -class EtlRun(Base): - """ - Information about an ETL run. - """ - __tablename__ = 'etl_run' - - id = Column(Integer(), primary_key=True, nullable=False ) - started_at = Column(DateTime()) - completed_at = Column(DateTime()) - pipeline_name = Column(Text()) - status = Column(Text()) - records_ingested = Column(Integer()) - note = Column(Text()) - - - def __repr__(self): - return f"EtlRun(id={self.id},started_at={self.started_at},completed_at={self.completed_at},pipeline_name={self.pipeline_name},status={self.status},records_ingested={self.records_ingested},note={self.note},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py deleted file mode 100644 index 85ddde97..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/methods_parameters_units.py +++ /dev/null @@ -1,1030 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/people.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/people.py deleted file mode 100644 index 8219be89..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/people.py +++ /dev/null @@ -1,106 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/places.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/places.py deleted file mode 100644 index 4eccd8ac..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/places.py +++ /dev/null @@ -1,106 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py deleted file mode 100644 index dfa0f14f..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/primary_ag_product.py +++ /dev/null @@ -1,75 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py deleted file mode 100644 index c02c738f..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_availability.py +++ /dev/null @@ -1,308 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer(), ForeignKey('Resource.id')) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py deleted file mode 100644 index b47201f8..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_class.py +++ /dev/null @@ -1,74 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py deleted file mode 100644 index 4568cca1..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_counterfactual.py +++ /dev/null @@ -1,1030 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer(), ForeignKey('Resource.id')) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py deleted file mode 100644 index 627f9124..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_information.py +++ /dev/null @@ -1,308 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py deleted file mode 100644 index 0b9bcf94..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_morphology.py +++ /dev/null @@ -1,308 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer(), ForeignKey('Resource.id')) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py deleted file mode 100644 index a3919852..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/resource_subclass.py +++ /dev/null @@ -1,74 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py deleted file mode 100644 index ae68592c..00000000 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/sample_preparation.py +++ /dev/null @@ -1,1119 +0,0 @@ - -from sqlalchemy import Column, Index, Table, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base -from sqlalchemy.ext.associationproxy import association_proxy - -Base = declarative_base() -metadata = Base.metadata - - -class BaseEntity(Base): - """ - Base entity included in all main entity tables. - """ - __tablename__ = 'base_entity' - - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"BaseEntity(id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - - -class LookupBase(Base): - """ - Base class for enum/ontology-like tables. - """ - __tablename__ = 'lookup_base' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LookupBase(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - - -class Geography(Base): - """ - Geographic location. - """ - __tablename__ = 'geography' - - geoid = Column(Text(), primary_key=True, nullable=False ) - state_name = Column(Text()) - state_fips = Column(Text()) - county_name = Column(Text()) - county_fips = Column(Text()) - region_name = Column(Text()) - agg_level_desc = Column(Text()) - - - def __repr__(self): - return f"Geography(geoid={self.geoid},state_name={self.state_name},state_fips={self.state_fips},county_name={self.county_name},county_fips={self.county_fips},region_name={self.region_name},agg_level_desc={self.agg_level_desc},)" - - - - - - -class ResourceMorphology(Base): - """ - Morphology of a resource. - """ - __tablename__ = 'resource_morphology' - - id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer()) - morphology_uri = Column(Text()) - - - def __repr__(self): - return f"ResourceMorphology(id={self.id},resource_id={self.resource_id},morphology_uri={self.morphology_uri},)" - - - - - - -class ParameterCategoryParameter(Base): - """ - Link between Parameter and ParameterCategory. - """ - __tablename__ = 'parameter_category_parameter' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - parameter_category_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterCategoryParameter(id={self.id},parameter_id={self.parameter_id},parameter_category_id={self.parameter_category_id},)" - - - - - - -class ParameterUnit(Base): - """ - Link between Parameter and Unit (alternate units). - """ - __tablename__ = 'parameter_unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer()) - alternate_unit_id = Column(Integer()) - - - def __repr__(self): - return f"ParameterUnit(id={self.id},parameter_id={self.parameter_id},alternate_unit_id={self.alternate_unit_id},)" - - - - - - -class PreparationMethod(BaseEntity): - """ - Method of sample preparation. - """ - __tablename__ = 'preparation_method' - - name = Column(Text()) - description = Column(Text()) - prep_method_abbrev_id = Column(Integer()) - prep_temp_c = Column(Numeric()) - uri = Column(Text()) - drying_step = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparationMethod(name={self.name},description={self.description},prep_method_abbrev_id={self.prep_method_abbrev_id},prep_temp_c={self.prep_temp_c},uri={self.uri},drying_step={self.drying_step},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparationMethodAbbreviation(LookupBase): - """ - Abbreviation for preparation method. - """ - __tablename__ = 'preparation_method_abbreviation' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PreparationMethodAbbreviation(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PreparedSample(BaseEntity): - """ - Sample that has been prepared. - """ - __tablename__ = 'prepared_sample' - - name = Column(Text()) - field_sample_id = Column(Integer()) - prep_method_id = Column(Integer()) - prep_date = Column(Date()) - preparer_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PreparedSample(name={self.name},field_sample_id={self.field_sample_id},prep_method_id={self.prep_method_id},prep_date={self.prep_date},preparer_id={self.preparer_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationAddress(BaseEntity): - """ - Physical address. - """ - __tablename__ = 'location_address' - - geography_id = Column(Text()) - address_line1 = Column(Text()) - address_line2 = Column(Text()) - city = Column(Text()) - zip = Column(Text()) - lat = Column(Float()) - lon = Column(Float()) - is_anonymous = Column(Boolean()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationAddress(geography_id={self.geography_id},address_line1={self.address_line1},address_line2={self.address_line2},city={self.city},zip={self.zip},lat={self.lat},lon={self.lon},is_anonymous={self.is_anonymous},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Contact(BaseEntity): - """ - Contact information for a person. - """ - __tablename__ = 'contact' - - first_name = Column(Text()) - last_name = Column(Text()) - email = Column(Text()) - affiliation = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Contact(first_name={self.first_name},last_name={self.last_name},email={self.email},affiliation={self.affiliation},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Provider(BaseEntity): - """ - Provider information. - """ - __tablename__ = 'provider' - - codename = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Provider(codename={self.codename},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Resource(BaseEntity): - """ - Biomass resource definition. - """ - __tablename__ = 'resource' - - name = Column(Text()) - primary_ag_product_id = Column(Integer()) - resource_class_id = Column(Integer()) - resource_subclass_id = Column(Integer()) - note = Column(Text()) - test = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Resource(name={self.name},primary_ag_product_id={self.primary_ag_product_id},resource_class_id={self.resource_class_id},resource_subclass_id={self.resource_subclass_id},note={self.note},test={self.test},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceClass(LookupBase): - """ - Classification of resources. - """ - __tablename__ = 'resource_class' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceClass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceSubclass(LookupBase): - """ - Sub-classification of resources. - """ - __tablename__ = 'resource_subclass' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ResourceSubclass(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PrimaryAgProduct(LookupBase): - """ - Primary agricultural product definition. - """ - __tablename__ = 'primary_ag_product' - - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"PrimaryAgProduct(note={self.note},id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceAvailability(BaseEntity): - """ - Availability of a resource in a location. - """ - __tablename__ = 'resource_availability' - - resource_id = Column(Integer()) - geoid = Column(Text()) - from_month = Column(Integer()) - to_month = Column(Integer()) - year_round = Column(Boolean()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceAvailability(resource_id={self.resource_id},geoid={self.geoid},from_month={self.from_month},to_month={self.to_month},year_round={self.year_round},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ResourceCounterfactual(BaseEntity): - """ - Counterfactual uses of a resource. - """ - __tablename__ = 'resource_counterfactual' - - geoid = Column(Text()) - resource_id = Column(Integer()) - counterfactual_description = Column(Text()) - animal_bedding_percent = Column(Numeric()) - animal_bedding_source_id = Column(Integer()) - animal_feed_percent = Column(Numeric()) - animal_feed_source_id = Column(Integer()) - bioelectricty_percent = Column(Numeric()) - bioelectricty_source_id = Column(Integer()) - burn_percent = Column(Numeric()) - burn_source_id = Column(Integer()) - compost_percent = Column(Numeric()) - compost_source_id = Column(Integer()) - landfill_percent = Column(Numeric()) - landfill_source_id = Column(Integer()) - counterfactual_date = Column(Date()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"ResourceCounterfactual(geoid={self.geoid},resource_id={self.resource_id},counterfactual_description={self.counterfactual_description},animal_bedding_percent={self.animal_bedding_percent},animal_bedding_source_id={self.animal_bedding_source_id},animal_feed_percent={self.animal_feed_percent},animal_feed_source_id={self.animal_feed_source_id},bioelectricty_percent={self.bioelectricty_percent},bioelectricty_source_id={self.bioelectricty_source_id},burn_percent={self.burn_percent},burn_source_id={self.burn_source_id},compost_percent={self.compost_percent},compost_source_id={self.compost_source_id},landfill_percent={self.landfill_percent},landfill_source_id={self.landfill_source_id},counterfactual_date={self.counterfactual_date},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSample(BaseEntity): - """ - Sample collected from the field. - """ - __tablename__ = 'field_sample' - - name = Column(Text()) - resource_id = Column(Integer()) - provider_id = Column(Integer()) - collector_id = Column(Integer()) - sample_collection_source = Column(Text()) - amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer()) - sampling_location_id = Column(Integer()) - field_storage_method_id = Column(Integer()) - field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer()) - field_storage_location_id = Column(Integer()) - collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer()) - harvest_method_id = Column(Integer()) - harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSample(name={self.name},resource_id={self.resource_id},provider_id={self.provider_id},collector_id={self.collector_id},sample_collection_source={self.sample_collection_source},amount_collected={self.amount_collected},amount_collected_unit_id={self.amount_collected_unit_id},sampling_location_id={self.sampling_location_id},field_storage_method_id={self.field_storage_method_id},field_storage_duration_value={self.field_storage_duration_value},field_storage_duration_unit_id={self.field_storage_duration_unit_id},field_storage_location_id={self.field_storage_location_id},collection_timestamp={self.collection_timestamp},collection_method_id={self.collection_method_id},harvest_method_id={self.harvest_method_id},harvest_date={self.harvest_date},field_sample_storage_location_id={self.field_sample_storage_location_id},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class PhysicalCharacteristic(BaseEntity): - """ - Physical characteristics of a sample. - """ - __tablename__ = 'physical_characteristic' - - field_sample_id = Column(Integer()) - particle_length = Column(Numeric()) - particle_width = Column(Numeric()) - particle_height = Column(Numeric()) - particle_unit_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"PhysicalCharacteristic(field_sample_id={self.field_sample_id},particle_length={self.particle_length},particle_width={self.particle_width},particle_height={self.particle_height},particle_unit_id={self.particle_unit_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldSampleCondition(BaseEntity): - """ - Condition of the field sample. - """ - __tablename__ = 'field_sample_condition' - - field_sample_id = Column(Integer()) - ag_treatment_id = Column(Integer()) - last_application_date = Column(Date()) - treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FieldSampleCondition(field_sample_id={self.field_sample_id},ag_treatment_id={self.ag_treatment_id},last_application_date={self.last_application_date},treatment_amount_per_acre={self.treatment_amount_per_acre},processing_method_id={self.processing_method_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FieldStorageMethod(LookupBase): - """ - Method of field storage. - """ - __tablename__ = 'field_storage_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"FieldStorageMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class CollectionMethod(LookupBase): - """ - Method of collection. - """ - __tablename__ = 'collection_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"CollectionMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class HarvestMethod(LookupBase): - """ - Method of harvest. - """ - __tablename__ = 'harvest_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"HarvestMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ProcessingMethod(LookupBase): - """ - Method of processing. - """ - __tablename__ = 'processing_method' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ProcessingMethod(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SoilType(LookupBase): - """ - Type of soil. - """ - __tablename__ = 'soil_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SoilType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class AgTreatment(LookupBase): - """ - Agricultural treatment. - """ - __tablename__ = 'ag_treatment' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"AgTreatment(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationSoilType(BaseEntity): - """ - Soil type at a location. - """ - __tablename__ = 'location_soil_type' - - location_id = Column(Integer()) - soil_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"LocationSoilType(location_id={self.location_id},soil_type_id={self.soil_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSource(BaseEntity): - """ - Source of data. - """ - __tablename__ = 'data_source' - - name = Column(Text()) - description = Column(Text()) - data_source_type_id = Column(Integer()) - full_title = Column(Text()) - creator = Column(Text()) - subject = Column(Text()) - publisher = Column(Text()) - contributor = Column(Text()) - date = Column(DateTime()) - type = Column(Text()) - biocirv = Column(Boolean()) - format = Column(Text()) - language = Column(Text()) - relation = Column(Text()) - temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer()) - rights = Column(Text()) - license = Column(Text()) - uri = Column(Text()) - note = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSource(name={self.name},description={self.description},data_source_type_id={self.data_source_type_id},full_title={self.full_title},creator={self.creator},subject={self.subject},publisher={self.publisher},contributor={self.contributor},date={self.date},type={self.type},biocirv={self.biocirv},format={self.format},language={self.language},relation={self.relation},temporal_coverage={self.temporal_coverage},location_coverage_id={self.location_coverage_id},rights={self.rights},license={self.license},uri={self.uri},note={self.note},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class FileObjectMetadata(BaseEntity): - """ - Metadata for a file object. - """ - __tablename__ = 'file_object_metadata' - - data_source_id = Column(Integer()) - bucket_path = Column(Text()) - file_format = Column(Text()) - file_size = Column(Integer()) - checksum_md5 = Column(Text()) - checksum_sha256 = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"FileObjectMetadata(data_source_id={self.data_source_id},bucket_path={self.bucket_path},file_format={self.file_format},file_size={self.file_size},checksum_md5={self.checksum_md5},checksum_sha256={self.checksum_sha256},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class DataSourceType(BaseEntity): - """ - Type of data source (e.g. database, literature). - """ - __tablename__ = 'data_source_type' - - source_type_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"DataSourceType(source_type_id={self.source_type_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class LocationResolution(LookupBase): - """ - Resolution of the location (e.g. nation, state, county). - """ - __tablename__ = 'location_resolution' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"LocationResolution(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class SourceType(LookupBase): - """ - Type of source (e.g. database, literature). - """ - __tablename__ = 'source_type' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"SourceType(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Unit(LookupBase): - """ - Unit of measurement. - """ - __tablename__ = 'unit' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"Unit(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Method(BaseEntity): - """ - Analytical method. - """ - __tablename__ = 'method' - - name = Column(Text()) - method_abbrev_id = Column(Integer()) - method_category_id = Column(Integer()) - method_standard_id = Column(Integer()) - description = Column(Text()) - detection_limits = Column(Text()) - source_id = Column(Integer()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Method(name={self.name},method_abbrev_id={self.method_abbrev_id},method_category_id={self.method_category_id},method_standard_id={self.method_standard_id},description={self.description},detection_limits={self.detection_limits},source_id={self.source_id},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodAbbrev(LookupBase): - """ - Abbreviation for method. - """ - __tablename__ = 'method_abbrev' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodAbbrev(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodCategory(LookupBase): - """ - Category of method. - """ - __tablename__ = 'method_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class MethodStandard(LookupBase): - """ - Standard associated with the method. - """ - __tablename__ = 'method_standard' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"MethodStandard(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class Parameter(BaseEntity): - """ - Parameter being measured. - """ - __tablename__ = 'parameter' - - name = Column(Text()) - standard_unit_id = Column(Integer()) - calculated = Column(Boolean()) - description = Column(Text()) - id = Column(Integer(), primary_key=True, nullable=False ) - created_at = Column(DateTime()) - updated_at = Column(DateTime()) - etl_run_id = Column(Text()) - lineage_group_id = Column(Integer()) - - - def __repr__(self): - return f"Parameter(name={self.name},standard_unit_id={self.standard_unit_id},calculated={self.calculated},description={self.description},id={self.id},created_at={self.created_at},updated_at={self.updated_at},etl_run_id={self.etl_run_id},lineage_group_id={self.lineage_group_id},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } - - - -class ParameterCategory(LookupBase): - """ - Category of parameter. - """ - __tablename__ = 'parameter_category' - - id = Column(Integer(), primary_key=True, nullable=False ) - name = Column(Text()) - description = Column(Text()) - uri = Column(Text()) - - - def __repr__(self): - return f"ParameterCategory(id={self.id},name={self.name},description={self.description},uri={self.uri},)" - - - - - # Using concrete inheritance: see https://docs.sqlalchemy.org/en/14/orm/inheritance.html - __mapper_args__ = { - 'concrete': True - } diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema1.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema1.py new file mode 100644 index 00000000..0fe4f136 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema1.py @@ -0,0 +1,22 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class ClassA(Base): + """ + + """ + __tablename__ = 'class_a' + + id = Column(Text(), primary_key=True, nullable=False ) + + + def __repr__(self): + return f"ClassA(id={self.id},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema2.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema2.py new file mode 100644 index 00000000..a5013e54 --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema2.py @@ -0,0 +1,40 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class ClassB(Base): + """ + + """ + __tablename__ = 'class_b' + + id = Column(Text(), primary_key=True, nullable=False ) + class_a_id = Column(Text(), ForeignKey('ClassA.id')) + + + def __repr__(self): + return f"ClassB(id={self.id},class_a_id={self.class_a_id},)" + + + + + + +class ClassA(Base): + """ + + """ + __tablename__ = 'class_a' + + id = Column(Text(), primary_key=True, nullable=False ) + + + def __repr__(self): + return f"ClassA(id={self.id},)" diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema3.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema3.py new file mode 100644 index 00000000..876e3f8f --- /dev/null +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated_test/schema3.py @@ -0,0 +1,59 @@ + +from sqlalchemy import Column, Index, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import * +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.associationproxy import association_proxy + +Base = declarative_base() +metadata = Base.metadata + + +class ClassC(Base): + """ + + """ + __tablename__ = 'class_c' + + id = Column(Text(), primary_key=True, nullable=False ) + class_a_id = Column(Text(), ForeignKey('ClassA.id')) + class_b_id = Column(Text(), ForeignKey('ClassB.id')) + + + def __repr__(self): + return f"ClassC(id={self.id},class_a_id={self.class_a_id},class_b_id={self.class_b_id},)" + + + + + + +class ClassA(Base): + """ + + """ + __tablename__ = 'class_a' + + id = Column(Text(), primary_key=True, nullable=False ) + + + def __repr__(self): + return f"ClassA(id={self.id},)" + + + + + + +class ClassB(Base): + """ + + """ + __tablename__ = 'class_b' + + id = Column(Text(), primary_key=True, nullable=False ) + class_a_id = Column(Text(), ForeignKey('ClassA.id')) + + + def __repr__(self): + return f"ClassB(id={self.id},class_a_id={self.class_a_id},)" diff --git a/src/ca_biositing/datamodels/utils/generate_sqla.py b/src/ca_biositing/datamodels/utils/generate_sqla.py index 50c0825f..929d5298 100644 --- a/src/ca_biositing/datamodels/utils/generate_sqla.py +++ b/src/ca_biositing/datamodels/utils/generate_sqla.py @@ -41,28 +41,13 @@ def generate_sqla(): # Ensure output directory exists output_dir.mkdir(parents=True, exist_ok=True) - # Clean output directory + # Clean output directory recursively print(f"Cleaning output directory: {output_dir}") - for file in output_dir.glob("*.py"): - if file.name != "__init__.py": - file.unlink() + for f in output_dir.glob('**/*.py'): + if f.name != '__init__.py': + f.unlink() # Generate for modules - for yaml_file in modules_dir.glob("*.yaml"): - module_name = yaml_file.stem - output_file = output_dir / f"{module_name}.py" - print(f"Generating {output_file} from {yaml_file}...") - - cmd = [ - "python", "-m", "linkml.generators.sqlalchemygen", - "--no-mergeimports", - str(yaml_file) - ] - - with open(output_file, "w") as f: - subprocess.run(cmd, stdout=f, check=True) - - post_process_file(output_file) # Generate for main schema main_yaml = linkml_dir / "ca_biositing.yaml" @@ -71,7 +56,7 @@ def generate_sqla(): cmd = [ "python", "-m", "linkml.generators.sqlalchemygen", - "--no-mergeimports", + "--mergeimports", str(main_yaml) ] diff --git a/src/ca_biositing/datamodels/utils/generate_test_sqla.py b/src/ca_biositing/datamodels/utils/generate_test_sqla.py new file mode 100644 index 00000000..c98f3184 --- /dev/null +++ b/src/ca_biositing/datamodels/utils/generate_test_sqla.py @@ -0,0 +1,69 @@ +import os +import re +import subprocess +from pathlib import Path + +def to_snake_case(name): + """Converts PascalCase to snake_case.""" + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + +def post_process_file(file_path): + """ + Post-processes the generated file to convert table names to snake_case. + """ + with open(file_path, 'r') as f: + content = f.read() + + # Regex to find __tablename__ assignments + # Matches: __tablename__ = 'SomeName' + pattern = r"__tablename__ = '([A-Za-z0-9]+)'" + + def replace_tablename(match): + original_name = match.group(1) + snake_name = to_snake_case(original_name) + return f"__tablename__ = '{snake_name}'" + + new_content = re.sub(pattern, replace_tablename, content) + + with open(file_path, 'w') as f: + f.write(new_content) + +def generate_sqla(): + """ + Generates SQLAlchemy models from LinkML schema modules. + """ + base_dir = Path(__file__).parent.parent + linkml_dir = base_dir / "ca_biositing/datamodels/linkml/test_schemas" + output_dir = base_dir / "ca_biositing/datamodels/schemas/generated_test" + + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + # Clean output directory + print(f"Cleaning output directory: {output_dir}") + for file in output_dir.glob("*.py"): + if file.name != "__init__.py": + file.unlink() + + # Generate for modules + for yaml_file in linkml_dir.glob("*.yaml"): + module_name = yaml_file.stem + output_file = output_dir / f"{module_name}.py" + print(f"Generating {output_file} from {yaml_file}...") + + cmd = [ + "python", "-m", "linkml.generators.sqlalchemygen", + "--mergeimports", + str(yaml_file) + ] + + with open(output_file, "w") as f: + subprocess.run(cmd, stdout=f, check=True) + + post_process_file(output_file) + + print("Generation complete.") + +if __name__ == "__main__": + generate_sqla() diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb index 140d8709..beb63faf 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -20,15 +20,15 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-06 10:31:31,052 - INFO - Added project root '/Users/pjsmitty301/ca-biositing' to sys.path\n", - "2026-01-06 10:31:32,479 - INFO - Successfully imported all project modules.\n" + "2026-01-06 10:39:39,502 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", + "2026-01-06 10:39:39,503 - INFO - Successfully imported all project modules.\n" ] } ], @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -193,6 +193,11 @@ " continue\n", " \n", " try:\n", + " # Skip normalization if the column is all NaN/None\n", + " if df_normalized[df_col].isnull().all():\n", + " logger.info(f\"Skipping normalization for column '{df_col}' as it contains only null values.\")\n", + " continue\n", + " \n", " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", " df_normalized, num_created = replace_name_with_id_df(\n", " db=db,\n", @@ -235,112 +240,109 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2026-01-06 10:31:40,834 - INFO - Starting data extraction...\n", - "2026-01-06 10:31:40,881 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:31:40,903 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:31:42,948 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/csrf-token?client=ae9812ec-079b-4fe5-845c-493121391421 \"HTTP/1.1 422 Unprocessable Entity\"\n", - "2026-01-06 10:31:42,966 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", - "2026-01-06 10:31:45,659 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:31:45,661 - INFO - Finished in state Completed()\n", - "2026-01-06 10:31:45,672 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:31:45,680 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:31:46,995 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", - "2026-01-06 10:31:47,292 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:31:47,296 - INFO - Finished in state Completed()\n", - "2026-01-06 10:31:47,313 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:31:47,326 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:31:48,924 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:31:48,929 - INFO - Finished in state Completed()\n", - "2026-01-06 10:31:48,930 - INFO - Data extraction complete.\n", - "2026-01-06 10:31:48,931 - INFO - Starting data cleaning...\n", - "2026-01-06 10:31:48,933 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:31:48,939 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-06 10:31:48,950 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:53:39,195 - INFO - Starting data extraction...\n", + "2026-01-06 10:53:39,212 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:53:39,266 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:53:41,468 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:53:41,471 - INFO - Finished in state Completed()\n", + "2026-01-06 10:53:41,486 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:53:41,497 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:53:42,816 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:53:42,820 - INFO - Finished in state Completed()\n", + "2026-01-06 10:53:42,840 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", + "2026-01-06 10:53:42,853 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", + "2026-01-06 10:53:44,427 - INFO - Successfully extracted raw data.\n", + "2026-01-06 10:53:44,430 - INFO - Finished in state Completed()\n", + "2026-01-06 10:53:44,434 - INFO - Data extraction complete.\n", + "2026-01-06 10:53:44,434 - INFO - Starting data cleaning...\n", + "2026-01-06 10:53:44,435 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:53:44,438 - INFO - Dropped 0 rows with missing values.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "2026-01-06 10:53:44,450 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:31:48,958 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:31:48,958 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:31:48,959 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-06 10:31:48,962 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:53:44,455 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:53:44,456 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:53:44,457 - INFO - Dropped 0 rows with missing values.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "2026-01-06 10:53:44,461 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:31:48,964 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:31:48,964 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:31:48,965 - INFO - Dropped 0 rows with missing values.\n", - "2026-01-06 10:31:48,969 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/183192953.py:42: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "2026-01-06 10:53:44,467 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:53:44,468 - INFO - Starting DataFrame cleaning process.\n", + "2026-01-06 10:53:44,470 - INFO - Dropped 0 rows with missing values.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "2026-01-06 10:53:44,480 - INFO - Successfully cleaned DataFrame.\n", + "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:31:48,972 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:31:48,973 - INFO - Data cleaning complete.\n", - "2026-01-06 10:31:48,973 - INFO - Starting data normalization...\n", - "2026-01-06 10:31:48,974 - INFO - Starting normalization process for 3 dataframes.\n", - "2026-01-06 10:31:48,974 - INFO - Processing DataFrame #1 with 1030 rows.\n", - "2026-01-06 10:31:48,974 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:31:49,016 - INFO - HTTP Request: POST http://127.0.0.1:4200/api/logs/ \"HTTP/1.1 201 Created\"\n", - "2026-01-06 10:31:49,116 - INFO - Created 35 new records in 'Resource' table.\n", - "2026-01-06 10:31:49,116 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-06 10:31:49,117 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:31:49,120 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-06 10:31:49,120 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:31:49,123 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-06 10:31:49,124 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:31:49,144 - INFO - Created 1 new records in 'Parameter' table.\n", - "2026-01-06 10:31:49,145 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-06 10:31:49,147 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:31:49,167 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:31:49,176 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-06 10:31:49,182 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-06 10:31:49,182 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:31:49,189 - INFO - Created 1 new records in 'AnalysisType' table.\n", - "2026-01-06 10:31:49,189 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-06 10:31:49,189 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", - "2026-01-06 10:31:49,190 - INFO - Finished processing DataFrame #1.\n", - "2026-01-06 10:31:49,190 - INFO - Processing DataFrame #2 with 64 rows.\n", - "2026-01-06 10:31:49,191 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:31:49,197 - INFO - Created 1 new records in 'Resource' table.\n", - "2026-01-06 10:31:49,198 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-06 10:31:49,198 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:31:49,201 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-06 10:31:49,202 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:31:49,204 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-06 10:31:49,204 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:31:49,206 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-06 10:31:49,206 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:31:49,209 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:31:49,209 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-06 10:31:49,211 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-06 10:31:49,212 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:31:49,213 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-06 10:31:49,213 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", - "2026-01-06 10:31:49,214 - INFO - Finished processing DataFrame #2.\n", - "2026-01-06 10:31:49,214 - INFO - Processing DataFrame #3 with 390 rows.\n", - "2026-01-06 10:31:49,215 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:31:49,217 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-06 10:31:49,217 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:31:49,220 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-06 10:31:49,220 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:31:49,222 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-06 10:31:49,222 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:31:49,224 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-06 10:31:49,224 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:31:49,225 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:31:49,225 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-06 10:31:49,228 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-06 10:31:49,228 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:31:49,230 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-06 10:31:49,230 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", - "2026-01-06 10:31:49,230 - INFO - Finished processing DataFrame #3.\n", - "2026-01-06 10:31:49,230 - INFO - Committing database session.\n", - "2026-01-06 10:31:49,232 - INFO - Database commit successful.\n", - "2026-01-06 10:31:49,241 - INFO - Data normalization complete.\n", - "2026-01-06 10:31:49,244 - INFO - Displaying results of normalization...\n" + "2026-01-06 10:53:44,487 - INFO - Converted all string data to lowercase.\n", + "2026-01-06 10:53:44,492 - INFO - Data cleaning complete.\n", + "2026-01-06 10:53:44,497 - INFO - Starting data normalization...\n", + "2026-01-06 10:53:44,498 - INFO - Starting normalization process for 3 dataframes.\n", + "2026-01-06 10:53:44,498 - INFO - Processing DataFrame #1 with 1030 rows.\n", + "2026-01-06 10:53:44,500 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:53:44,526 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 46 null values.\n", + "2026-01-06 10:53:44,527 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:53:44,531 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 22 null values.\n", + "2026-01-06 10:53:44,531 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:53:44,533 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 22 null values.\n", + "2026-01-06 10:53:44,534 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:53:44,536 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 22 null values.\n", + "2026-01-06 10:53:44,537 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:53:44,539 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:53:44,539 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:53:44,542 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 550 null values.\n", + "2026-01-06 10:53:44,543 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:53:44,546 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 550 null values.\n", + "2026-01-06 10:53:44,546 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", + "2026-01-06 10:53:44,546 - INFO - Finished processing DataFrame #1.\n", + "2026-01-06 10:53:44,546 - INFO - Processing DataFrame #2 with 64 rows.\n", + "2026-01-06 10:53:44,547 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:53:44,550 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:53:44,550 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:53:44,552 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:53:44,552 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:53:44,554 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:53:44,554 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:53:44,556 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:53:44,557 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:53:44,558 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:53:44,559 - INFO - Skipping normalization for column 'analyst_email' as it contains only null values.\n", + "2026-01-06 10:53:44,559 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:53:44,561 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:53:44,561 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", + "2026-01-06 10:53:44,561 - INFO - Finished processing DataFrame #2.\n", + "2026-01-06 10:53:44,562 - INFO - Processing DataFrame #3 with 390 rows.\n", + "2026-01-06 10:53:44,562 - INFO - Normalizing column 'resource' using model 'Resource'.\n", + "2026-01-06 10:53:44,567 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", + "2026-01-06 10:53:44,569 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", + "2026-01-06 10:53:44,580 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", + "2026-01-06 10:53:44,580 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", + "2026-01-06 10:53:44,592 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", + "2026-01-06 10:53:44,594 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", + "2026-01-06 10:53:44,597 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", + "2026-01-06 10:53:44,597 - INFO - Normalizing column 'unit' using model 'Unit'.\n", + "2026-01-06 10:53:44,598 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", + "2026-01-06 10:53:44,599 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", + "2026-01-06 10:53:44,606 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", + "2026-01-06 10:53:44,607 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", + "2026-01-06 10:53:44,610 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", + "2026-01-06 10:53:44,610 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", + "2026-01-06 10:53:44,610 - INFO - Finished processing DataFrame #3.\n", + "2026-01-06 10:53:44,611 - INFO - Committing database session.\n", + "2026-01-06 10:53:44,612 - INFO - Database commit successful.\n", + "2026-01-06 10:53:44,612 - INFO - Data normalization complete.\n", + "2026-01-06 10:53:44,613 - INFO - Displaying results of normalization...\n" ] }, { @@ -417,32 +419,32 @@ { "name": "upload_status", "rawType": "object", - "type": "string" + "type": "unknown" }, { "name": "note", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "resource_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" }, { "name": "prepared_sample_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" }, { "name": "preparation_method_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" }, { "name": "parameter_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" }, { "name": "unit_id", @@ -451,16 +453,16 @@ }, { "name": "analyst_email_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" }, { "name": "analysis_type_id", - "rawType": "int64", - "type": "integer" + "rawType": "float64", + "type": "float" } ], - "ref": "b0e10f3c-2f75-4628-acc4-1c383e10ca93", + "ref": "bcafe3ce-a742-48cc-9fb2-da62f6c91695", "rows": [ [ "0", @@ -476,14 +478,14 @@ null, "pass", "not ready", - "", - "6", - "109", - "7", - "3", - "1", + null, + "5.0", + "109.0", + "7.0", + "3.0", "1", - "1" + "1.0", + "1.0" ], [ "1", @@ -499,14 +501,14 @@ null, "pass", "ready", - "", - "6", - "109", - "7", - "3", - "1", + null, + "5.0", + "109.0", + "7.0", + "3.0", "1", - "1" + "1.0", + "1.0" ], [ "2", @@ -522,14 +524,14 @@ null, "pass", "imported", - "", - "6", - "109", - "7", - "3", - "1", + null, + "5.0", + "109.0", + "7.0", + "3.0", "1", - "1" + "1.0", + "1.0" ], [ "3", @@ -545,14 +547,14 @@ null, "pass", "import failed", - "", - "6", - "109", - "7", - "4", - "1", + null, + "5.0", + "109.0", + "7.0", + "4.0", "1", - "1" + "1.0", + "1.0" ], [ "4", @@ -567,15 +569,15 @@ "2024-10-03 10:31:31", null, "pass", - "", - "", - "6", - "109", - "7", - "4", - "1", + null, + null, + "5.0", + "109.0", + "7.0", + "4.0", "1", - "1" + "1.0", + "1.0" ] ], "shape": { @@ -639,14 +641,14 @@ " NaT\n", " pass\n", " not ready\n", - " \n", - " 6\n", - " 109\n", - " 7\n", - " 3\n", - " 1\n", - " 1\n", + " NaN\n", + " 5.0\n", + " 109.0\n", + " 7.0\n", + " 3.0\n", " 1\n", + " 1.0\n", + " 1.0\n", " \n", " \n", " 1\n", @@ -662,14 +664,14 @@ " NaT\n", " pass\n", " ready\n", - " \n", - " 6\n", - " 109\n", - " 7\n", - " 3\n", - " 1\n", - " 1\n", + " NaN\n", + " 5.0\n", + " 109.0\n", + " 7.0\n", + " 3.0\n", " 1\n", + " 1.0\n", + " 1.0\n", " \n", " \n", " 2\n", @@ -685,14 +687,14 @@ " NaT\n", " pass\n", " imported\n", - " \n", - " 6\n", - " 109\n", - " 7\n", - " 3\n", - " 1\n", - " 1\n", + " NaN\n", + " 5.0\n", + " 109.0\n", + " 7.0\n", + " 3.0\n", " 1\n", + " 1.0\n", + " 1.0\n", " \n", " \n", " 3\n", @@ -708,14 +710,14 @@ " NaT\n", " pass\n", " import failed\n", - " \n", - " 6\n", - " 109\n", - " 7\n", - " 4\n", - " 1\n", - " 1\n", + " NaN\n", + " 5.0\n", + " 109.0\n", + " 7.0\n", + " 4.0\n", " 1\n", + " 1.0\n", + " 1.0\n", " \n", " \n", " 4\n", @@ -730,15 +732,15 @@ " 2024-10-03 10:31:31\n", " NaT\n", " pass\n", - " \n", - " \n", - " 6\n", - " 109\n", - " 7\n", - " 4\n", - " 1\n", - " 1\n", + " <NA>\n", + " NaN\n", + " 5.0\n", + " 109.0\n", + " 7.0\n", + " 4.0\n", " 1\n", + " 1.0\n", + " 1.0\n", " \n", " \n", "\n", @@ -759,26 +761,26 @@ "3 prox01xk 1.0 prox01xk(73)1 0.690000 2024-10-03 10:31:01 \n", "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", "\n", - " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", - "0 NaT pass not ready 6 109 \n", - "1 NaT pass ready 6 109 \n", - "2 NaT pass imported 6 109 \n", - "3 NaT pass import failed 6 109 \n", - "4 NaT pass 6 109 \n", + " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", + "0 NaT pass not ready NaN 5.0 109.0 \n", + "1 NaT pass ready NaN 5.0 109.0 \n", + "2 NaT pass imported NaN 5.0 109.0 \n", + "3 NaT pass import failed NaN 5.0 109.0 \n", + "4 NaT pass NaN 5.0 109.0 \n", "\n", " preparation_method_id parameter_id unit_id analyst_email_id \\\n", - "0 7 3 1 1 \n", - "1 7 3 1 1 \n", - "2 7 3 1 1 \n", - "3 7 4 1 1 \n", - "4 7 4 1 1 \n", + "0 7.0 3.0 1 1.0 \n", + "1 7.0 3.0 1 1.0 \n", + "2 7.0 3.0 1 1.0 \n", + "3 7.0 4.0 1 1.0 \n", + "4 7.0 4.0 1 1.0 \n", "\n", " analysis_type_id \n", - "0 1 \n", - "1 1 \n", - "2 1 \n", - "3 1 \n", - "4 1 " + "0 1.0 \n", + "1 1.0 \n", + "2 1.0 \n", + "3 1.0 \n", + "4 1.0 " ] }, "metadata": {}, @@ -862,18 +864,23 @@ }, { "name": "equipment", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "raw_data_url", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" + }, + { + "name": "analyst_email", + "rawType": "float64", + "type": "float" }, { "name": "upload_status", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "resource_id", @@ -900,18 +907,13 @@ "rawType": "int64", "type": "integer" }, - { - "name": "analyst_email_id", - "rawType": "int64", - "type": "integer" - }, { "name": "analysis_type_id", "rawType": "int64", "type": "integer" } ], - "ref": "0f732e49-22cb-4b72-a52a-9fade9af05ef", + "ref": "d0538baa-0f16-4d3e-992f-e11d99faffd6", "rows": [ [ "0", @@ -927,15 +929,15 @@ null, "pass", "1", - "", - "", - "", - "27", + null, + null, + null, + null, + "26", "80", "6", "8", "3", - "2", "2" ], [ @@ -952,15 +954,15 @@ null, "fail", " 1 dup", - "", - "", - "", - "27", + null, + null, + null, + null, + "26", "80", "6", "8", "3", - "2", "2" ], [ @@ -977,15 +979,15 @@ null, "pass", "2", - "", - "", - "", - "11", + null, + null, + null, + null, + "10", "96", "6", "8", "3", - "2", "2" ], [ @@ -1002,15 +1004,15 @@ null, "pass", "3", - "", - "", - "", - "12", + null, + null, + null, + null, + "11", "91", "9", "8", "3", - "2", "2" ], [ @@ -1027,15 +1029,15 @@ null, "pass", "4", - "", - "", - "", - "21", + null, + null, + null, + null, + "20", "153", "9", "8", "3", - "2", "2" ] ], @@ -1076,13 +1078,13 @@ " ...\n", " equipment\n", " raw_data_url\n", + " analyst_email\n", " upload_status\n", " resource_id\n", " prepared_sample_id\n", " preparation_method_id\n", " parameter_id\n", " unit_id\n", - " analyst_email_id\n", " analysis_type_id\n", " \n", " \n", @@ -1100,16 +1102,16 @@ " NaT\n", " NaT\n", " ...\n", - " \n", - " \n", - " \n", - " 27\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 26\n", " 80\n", " 6\n", " 8\n", " 3\n", " 2\n", - " 2\n", " \n", " \n", " 1\n", @@ -1124,16 +1126,16 @@ " NaT\n", " NaT\n", " ...\n", - " \n", - " \n", - " \n", - " 27\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 26\n", " 80\n", " 6\n", " 8\n", " 3\n", " 2\n", - " 2\n", " \n", " \n", " 2\n", @@ -1148,16 +1150,16 @@ " NaT\n", " NaT\n", " ...\n", - " \n", - " \n", - " \n", - " 11\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 10\n", " 96\n", " 6\n", " 8\n", " 3\n", " 2\n", - " 2\n", " \n", " \n", " 3\n", @@ -1172,16 +1174,16 @@ " NaT\n", " NaT\n", " ...\n", - " \n", - " \n", - " \n", - " 12\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 11\n", " 91\n", " 9\n", " 8\n", " 3\n", " 2\n", - " 2\n", " \n", " \n", " 4\n", @@ -1196,16 +1198,16 @@ " NaT\n", " NaT\n", " ...\n", - " \n", - " \n", - " \n", - " 21\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 20\n", " 153\n", " 9\n", " 8\n", " 3\n", " 2\n", - " 2\n", " \n", " \n", "\n", @@ -1227,26 +1229,26 @@ "3 rt vacuum sealed ult26kh 1 ult26kh(82)1 92.199997 NaT \n", "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", "\n", - " updated_at ... equipment raw_data_url upload_status resource_id \\\n", - "0 NaT ... 27 \n", - "1 NaT ... 27 \n", - "2 NaT ... 11 \n", - "3 NaT ... 12 \n", - "4 NaT ... 21 \n", + " updated_at ... equipment raw_data_url analyst_email upload_status \\\n", + "0 NaT ... NaN NaN NaN NaN \n", + "1 NaT ... NaN NaN NaN NaN \n", + "2 NaT ... NaN NaN NaN NaN \n", + "3 NaT ... NaN NaN NaN NaN \n", + "4 NaT ... NaN NaN NaN NaN \n", "\n", - " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 80 6 8 3 \n", - "1 80 6 8 3 \n", - "2 96 6 8 3 \n", - "3 91 9 8 3 \n", - "4 153 9 8 3 \n", + " resource_id prepared_sample_id preparation_method_id parameter_id \\\n", + "0 26 80 6 8 \n", + "1 26 80 6 8 \n", + "2 10 96 6 8 \n", + "3 11 91 9 8 \n", + "4 20 153 9 8 \n", "\n", - " analyst_email_id analysis_type_id \n", - "0 2 2 \n", - "1 2 2 \n", - "2 2 2 \n", - "3 2 2 \n", - "4 2 2 \n", + " unit_id analysis_type_id \n", + "0 3 2 \n", + "1 3 2 \n", + "2 3 2 \n", + "3 3 2 \n", + "4 3 2 \n", "\n", "[5 rows x 22 columns]" ] @@ -1322,18 +1324,18 @@ }, { "name": "note", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "equipment", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "raw_data_url", - "rawType": "object", - "type": "string" + "rawType": "float64", + "type": "float" }, { "name": "upload_status", @@ -1376,7 +1378,7 @@ "type": "integer" } ], - "ref": "95361c64-4a65-4fd6-bde6-2efae692d84c", + "ref": "1f353831-a3bd-4f43-a253-769adc8b66eb", "rows": [ [ "0", @@ -1390,11 +1392,11 @@ "2025-01-23 09:00:01", null, "pass", - "", - "", - "", + null, + null, + null, "ready", - "6", + "5", "92", "8", "13", @@ -1414,11 +1416,11 @@ "2025-01-23 09:00:16", null, "pass", - "", - "", - "", + null, + null, + null, "ready", - "6", + "5", "92", "8", "13", @@ -1438,11 +1440,11 @@ "2025-01-23 09:00:31", null, "pass", - "", - "", - "", + null, + null, + null, "ready", - "6", + "5", "92", "8", "13", @@ -1462,11 +1464,11 @@ "2025-01-23 09:00:46", null, "pass", - "", - "", - "", + null, + null, + null, "ready", - "6", + "5", "92", "8", "11", @@ -1486,11 +1488,11 @@ "2025-01-23 09:01:01", null, "pass", - "", - "", - "", + null, + null, + null, "ready", - "6", + "5", "92", "8", "11", @@ -1560,10 +1562,10 @@ " NaT\n", " pass\n", " ...\n", - " \n", - " \n", + " NaN\n", + " NaN\n", " ready\n", - " 6\n", + " 5\n", " 92\n", " 8\n", " 13\n", @@ -1584,10 +1586,10 @@ " NaT\n", " pass\n", " ...\n", - " \n", - " \n", + " NaN\n", + " NaN\n", " ready\n", - " 6\n", + " 5\n", " 92\n", " 8\n", " 13\n", @@ -1608,10 +1610,10 @@ " NaT\n", " pass\n", " ...\n", - " \n", - " \n", + " NaN\n", + " NaN\n", " ready\n", - " 6\n", + " 5\n", " 92\n", " 8\n", " 13\n", @@ -1632,10 +1634,10 @@ " NaT\n", " pass\n", " ...\n", - " \n", - " \n", + " NaN\n", + " NaN\n", " ready\n", - " 6\n", + " 5\n", " 92\n", " 8\n", " 11\n", @@ -1656,10 +1658,10 @@ " NaT\n", " pass\n", " ...\n", - " \n", - " \n", + " NaN\n", + " NaN\n", " ready\n", - " 6\n", + " 5\n", " 92\n", " 8\n", " 11\n", @@ -1687,12 +1689,12 @@ "3 cmp04xk 1 cmp04xk(85)1 15.74 2025-01-23 09:00:46 NaT \n", "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", "\n", - " qc_result ... equipment raw_data_url upload_status resource_id \\\n", - "0 pass ... ready 6 \n", - "1 pass ... ready 6 \n", - "2 pass ... ready 6 \n", - "3 pass ... ready 6 \n", - "4 pass ... ready 6 \n", + " qc_result ... equipment raw_data_url upload_status resource_id \\\n", + "0 pass ... NaN NaN ready 5 \n", + "1 pass ... NaN NaN ready 5 \n", + "2 pass ... NaN NaN ready 5 \n", + "3 pass ... NaN NaN ready 5 \n", + "4 pass ... NaN NaN ready 5 \n", "\n", " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", "0 92 8 13 2 \n", From 234a235b182d454648fb9f5b4d64e29d44402b2f Mon Sep 17 00:00:00 2001 From: petercarbsmith Date: Wed, 7 Jan 2026 14:14:07 -0700 Subject: [PATCH 09/10] fixed the alembic import problems. models now have fk --- alembic/env.py | 25 +- .../04f37f9dbb48_test_test_test_test_test.py | 1602 +++++++++++++++++ .../1ace3fc9c593_initial_migration.py | 32 + ...b18e6e5e72d_trying_with_fixing_the_base.py | 32 + ...5d_fixed_alembic_environment_to_import_.py | 32 + ..._fixed_alembic_gen_sqla_so_all_inherit_.py | 32 + ...9be24ffa7bb9_refactor_and_update_env_py.py | 1462 +++++++++++++++ ...88_fixed_alembic_environment_to_import_.py | 32 + .../ca_biositing/datamodels/database.py | 20 +- .../schemas/generated/ca_biositing.py | 342 ++-- .../datamodels/utils/generate_sqla.py | 31 + 11 files changed, 3448 insertions(+), 194 deletions(-) create mode 100644 alembic/versions/04f37f9dbb48_test_test_test_test_test.py create mode 100644 alembic/versions/1ace3fc9c593_initial_migration.py create mode 100644 alembic/versions/2b18e6e5e72d_trying_with_fixing_the_base.py create mode 100644 alembic/versions/2b41e390fc5d_fixed_alembic_environment_to_import_.py create mode 100644 alembic/versions/62ef89b0050a_fixed_alembic_gen_sqla_so_all_inherit_.py create mode 100644 alembic/versions/9be24ffa7bb9_refactor_and_update_env_py.py create mode 100644 alembic/versions/a8a3ea954e88_fixed_alembic_environment_to_import_.py diff --git a/alembic/env.py b/alembic/env.py index 068e73bb..ad465742 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -24,16 +24,12 @@ # from ca_biositing.datamodels.sample_preprocessing import * # from ca_biositing.datamodels.specific_aalysis_results import * # from ca_biositing.datamodels.user import * +from ca_biositing.datamodels.schemas.generated import ca_biositing from ca_biositing.datamodels.schemas.generated.ca_biositing import * -from sqlmodel import SQLModel -import importlib.util -from pathlib import Path - -# --- Import generated models and their metadata --- -# from ca_biositing.datamodels.schemas.generated.census_survey import metadata as census_metadata -# from ca_biositing.datamodels.schemas.generated.geography import metadata as geography_metadata +from ca_biositing.datamodels.database import Base -# --- Alembic Config object, provides access to alembic.ini values --- +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. config = context.config # Override sqlalchemy.url in alembic.ini with value from .env @@ -43,16 +39,15 @@ else: raise RuntimeError("DATABASE_URL not found in .env file. Alembic cannot run migrations.") -# --- Configure logging (from alembic.ini logging section) --- +# Interpret the config file for Python logging. +# This line sets up loggers basically. if config.config_file_name is not None: fileConfig(config.config_file_name) -# --- Metadata from your models for autogenerate --- -# Combine metadata from all models -# See: https://alembic.sqlalchemy.org/en/latest/autogenerate.html#affecting-the-autogenerate-process - -# Start with the default SQLModel metadata -# target_metadata = SQLModel.metadata +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata target_metadata = Base.metadata # Manually merge metadata from generated modules diff --git a/alembic/versions/04f37f9dbb48_test_test_test_test_test.py b/alembic/versions/04f37f9dbb48_test_test_test_test_test.py new file mode 100644 index 00000000..2ccc998f --- /dev/null +++ b/alembic/versions/04f37f9dbb48_test_test_test_test_test.py @@ -0,0 +1,1602 @@ +"""test test test test test + +Revision ID: 04f37f9dbb48 +Revises: 2b18e6e5e72d +Create Date: 2026-01-07 21:06:50.957245 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '04f37f9dbb48' +down_revision: Union[str, Sequence[str], None] = '2b18e6e5e72d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('ag_treatment', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('analysis_type', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('base_entity', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('collection_method', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('contact', + sa.Column('first_name', sa.Text(), nullable=True), + sa.Column('last_name', sa.Text(), nullable=True), + sa.Column('email', sa.Text(), nullable=True), + sa.Column('affiliation', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('dimension_type', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('entity_lineage', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.Column('source_table', sa.Text(), nullable=True), + sa.Column('source_row_id', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('etl_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('pipeline_name', sa.Text(), nullable=True), + sa.Column('status', sa.Text(), nullable=True), + sa.Column('records_ingested', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('field_storage_method', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('geography', + sa.Column('geoid', sa.Text(), nullable=False), + sa.Column('state_name', sa.Text(), nullable=True), + sa.Column('state_fips', sa.Text(), nullable=True), + sa.Column('county_name', sa.Text(), nullable=True), + sa.Column('county_fips', sa.Text(), nullable=True), + sa.Column('region_name', sa.Text(), nullable=True), + sa.Column('agg_level_desc', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('geoid') + ) + op.create_table('harvest_method', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('infrastructure_biodiesel_plants', + sa.Column('biodiesel_plant_id', sa.Integer(), nullable=False), + sa.Column('company', sa.Text(), nullable=True), + sa.Column('bbi_index', sa.Integer(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('capacity_mmg_per_y', sa.Integer(), nullable=True), + sa.Column('feedstock', sa.Text(), nullable=True), + sa.Column('status', sa.Text(), nullable=True), + sa.Column('address', sa.Text(), nullable=True), + sa.Column('coordinates', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.Column('source', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('biodiesel_plant_id') + ) + op.create_table('infrastructure_biosolids_facilities', + sa.Column('biosolid_facility_id', sa.Integer(), nullable=False), + sa.Column('report_submitted_date', sa.Date(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.Column('facility', sa.Text(), nullable=True), + sa.Column('authority', sa.Text(), nullable=True), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.Column('aqmd', sa.Text(), nullable=True), + sa.Column('facility_address', sa.Text(), nullable=True), + sa.Column('facility_city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('facility_zip', sa.Text(), nullable=True), + sa.Column('facility_county', sa.Text(), nullable=True), + sa.Column('mailing_street_1', sa.Text(), nullable=True), + sa.Column('mailing_city', sa.Text(), nullable=True), + sa.Column('mailing_state', sa.Text(), nullable=True), + sa.Column('mailing_zip', sa.Text(), nullable=True), + sa.Column('biosolids_number', sa.Text(), nullable=True), + sa.Column('biosolids_contact', sa.Text(), nullable=True), + sa.Column('biosolids_contact_phone', sa.Text(), nullable=True), + sa.Column('biosolids_contact_email', sa.Text(), nullable=True), + sa.Column('adwf', sa.Numeric(), nullable=True), + sa.Column('potw_biosolids_generated', sa.Integer(), nullable=True), + sa.Column('twtds_biosolids_treated', sa.Integer(), nullable=True), + sa.Column('class_b_land_app', sa.Integer(), nullable=True), + sa.Column('class_b_applier', sa.Text(), nullable=True), + sa.Column('class_a_compost', sa.Integer(), nullable=True), + sa.Column('class_a_heat_dried', sa.Integer(), nullable=True), + sa.Column('class_a_other', sa.Integer(), nullable=True), + sa.Column('class_a_other_applier', sa.Text(), nullable=True), + sa.Column('twtds_transfer_to_second_preparer', sa.Integer(), nullable=True), + sa.Column('twtds_second_preparer_name', sa.Text(), nullable=True), + sa.Column('adc_or_final_c', sa.Integer(), nullable=True), + sa.Column('landfill', sa.Integer(), nullable=True), + sa.Column('landfill_name', sa.Text(), nullable=True), + sa.Column('surface_disposal', sa.Integer(), nullable=True), + sa.Column('deepwell_injection', sa.Text(), nullable=True), + sa.Column('stored', sa.Integer(), nullable=True), + sa.Column('longterm_treatment', sa.Integer(), nullable=True), + sa.Column('other', sa.Integer(), nullable=True), + sa.Column('name_of_other', sa.Text(), nullable=True), + sa.Column('incineration', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('biosolid_facility_id') + ) + op.create_table('infrastructure_cafo_manure_locations', + sa.Column('cafo_manure_id', sa.Integer(), nullable=False), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.Column('owner_name', sa.Text(), nullable=True), + sa.Column('facility_name', sa.Text(), nullable=True), + sa.Column('address', sa.Text(), nullable=True), + sa.Column('town', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('zip', sa.Text(), nullable=True), + sa.Column('animal', sa.Text(), nullable=True), + sa.Column('animal_feed_operation_type', sa.Text(), nullable=True), + sa.Column('animal_units', sa.Integer(), nullable=True), + sa.Column('animal_count', sa.Integer(), nullable=True), + sa.Column('manure_total_solids', sa.Numeric(), nullable=True), + sa.Column('source', sa.Text(), nullable=True), + sa.Column('date_accessed', sa.Date(), nullable=True), + sa.PrimaryKeyConstraint('cafo_manure_id') + ) + op.create_table('infrastructure_combustion_plants', + sa.Column('combustion_fid', sa.Integer(), nullable=False), + sa.Column('objectid', sa.Integer(), nullable=True), + sa.Column('status', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True), + sa.Column('equivalent_generation', sa.Numeric(), nullable=True), + sa.Column('np_mw', sa.Numeric(), nullable=True), + sa.Column('cf', sa.Numeric(), nullable=True), + sa.Column('yearload', sa.Integer(), nullable=True), + sa.Column('fuel', sa.Text(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('wkt_geom', sa.Text(), nullable=True), + sa.Column('geom', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('combustion_fid') + ) + op.create_table('infrastructure_district_energy_systems', + sa.Column('des_fid', sa.Integer(), nullable=False), + sa.Column('cbg_id', sa.Integer(), nullable=True), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('system', sa.Text(), nullable=True), + sa.Column('object_id', sa.Integer(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('primary_fuel', sa.Text(), nullable=True), + sa.Column('secondary_fuel', sa.Text(), nullable=True), + sa.Column('usetype', sa.Text(), nullable=True), + sa.Column('cap_st', sa.Numeric(), nullable=True), + sa.Column('cap_hw', sa.Numeric(), nullable=True), + sa.Column('cap_cw', sa.Numeric(), nullable=True), + sa.Column('chpcg_cap', sa.Numeric(), nullable=True), + sa.Column('excess_c', sa.Numeric(), nullable=True), + sa.Column('excess_h', sa.Numeric(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('wkt_geom', sa.Text(), nullable=True), + sa.Column('geom', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('des_fid') + ) + op.create_table('infrastructure_ethanol_biorefineries', + sa.Column('ethanol_biorefinery_id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('address', sa.Text(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('capacity_mgy', sa.Integer(), nullable=True), + sa.Column('production_mgy', sa.Integer(), nullable=True), + sa.Column('constr_exp', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('ethanol_biorefinery_id') + ) + op.create_table('infrastructure_food_processing_facilities', + sa.Column('processing_facility_id', sa.Integer(), nullable=False), + sa.Column('address', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('company', sa.Text(), nullable=True), + sa.Column('join_count', sa.Integer(), nullable=True), + sa.Column('master_type', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('subtype', sa.Text(), nullable=True), + sa.Column('target_fid', sa.Integer(), nullable=True), + sa.Column('processing_type', sa.Text(), nullable=True), + sa.Column('zip', sa.Text(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('wkt_geom', sa.Text(), nullable=True), + sa.Column('geom', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('processing_facility_id') + ) + op.create_table('infrastructure_landfills', + sa.Column('project_id', sa.Text(), nullable=False), + sa.Column('project_int_id', sa.Integer(), nullable=True), + sa.Column('ghgrp_id', sa.Text(), nullable=True), + sa.Column('landfill_id', sa.Integer(), nullable=True), + sa.Column('landfill_name', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('physical_address', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True), + sa.Column('zip_code', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.Column('ownership_type', sa.Text(), nullable=True), + sa.Column('landfill_owner_orgs', sa.Text(), nullable=True), + sa.Column('landfill_opened_year', sa.Date(), nullable=True), + sa.Column('landfill_closure_year', sa.Date(), nullable=True), + sa.Column('landfill_status', sa.Text(), nullable=True), + sa.Column('waste_in_place', sa.Integer(), nullable=True), + sa.Column('waste_in_place_year', sa.Date(), nullable=True), + sa.Column('lfg_system_in_place', sa.Boolean(), nullable=True), + sa.Column('lfg_collected', sa.Numeric(), nullable=True), + sa.Column('lfg_flared', sa.Numeric(), nullable=True), + sa.Column('project_status', sa.Text(), nullable=True), + sa.Column('project_name', sa.Text(), nullable=True), + sa.Column('project_start_date', sa.Date(), nullable=True), + sa.Column('project_shutdown_date', sa.Date(), nullable=True), + sa.Column('project_type_category', sa.Text(), nullable=True), + sa.Column('lfg_energy_project_type', sa.Text(), nullable=True), + sa.Column('rng_delivery_method', sa.Text(), nullable=True), + sa.Column('actual_mw_generation', sa.Numeric(), nullable=True), + sa.Column('rated_mw_capacity', sa.Numeric(), nullable=True), + sa.Column('lfg_flow_to_project', sa.Numeric(), nullable=True), + sa.Column('direct_emission_reductions', sa.Numeric(), nullable=True), + sa.Column('avoided_emission_reductions', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('project_id') + ) + op.create_table('infrastructure_livestock_anaerobic_digesters', + sa.Column('digester_id', sa.Integer(), nullable=False), + sa.Column('project_name', sa.Text(), nullable=True), + sa.Column('project_type', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('digester_type', sa.Text(), nullable=True), + sa.Column('profile', sa.Text(), nullable=True), + sa.Column('year_operational', sa.Date(), nullable=True), + sa.Column('animal_type_class', sa.Text(), nullable=True), + sa.Column('animal_types', sa.Text(), nullable=True), + sa.Column('pop_feeding_digester', sa.Text(), nullable=True), + sa.Column('total_pop_feeding_digester', sa.Integer(), nullable=True), + sa.Column('cattle', sa.Integer(), nullable=True), + sa.Column('dairy', sa.Integer(), nullable=True), + sa.Column('poultry', sa.Integer(), nullable=True), + sa.Column('swine', sa.Integer(), nullable=True), + sa.Column('codigestion', sa.Text(), nullable=True), + sa.Column('biogas_generation_estimate', sa.Integer(), nullable=True), + sa.Column('electricity_generated', sa.Integer(), nullable=True), + sa.Column('biogas_end_uses', sa.Text(), nullable=True), + sa.Column('methane_emission_reductions', sa.Integer(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('digester_id') + ) + op.create_table('infrastructure_msw_to_energy_anaerobic_digesters', + sa.Column('wte_id', sa.Integer(), nullable=False), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True), + sa.Column('equivalent_generation', sa.Numeric(), nullable=True), + sa.Column('feedstock', sa.Text(), nullable=True), + sa.Column('dayload', sa.Numeric(), nullable=True), + sa.Column('dayload_bdt', sa.Numeric(), nullable=True), + sa.Column('facility_type', sa.Text(), nullable=True), + sa.Column('status', sa.Text(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('source', sa.Text(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('wkt_geom', sa.Text(), nullable=True), + sa.Column('geom', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('wte_id') + ) + op.create_table('infrastructure_saf_and_renewable_diesel_plants', + sa.Column('ibcc_index', sa.Integer(), nullable=False), + sa.Column('company', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('country', sa.Text(), nullable=True), + sa.Column('capacity', sa.Text(), nullable=True), + sa.Column('feedstock', sa.Text(), nullable=True), + sa.Column('products', sa.Text(), nullable=True), + sa.Column('status', sa.Text(), nullable=True), + sa.Column('address', sa.Text(), nullable=True), + sa.Column('coordinates', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.PrimaryKeyConstraint('ibcc_index') + ) + op.create_table('infrastructure_wastewater_treatment_plants', + sa.Column('plant_id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True), + sa.Column('codigestion', sa.Text(), nullable=True), + sa.Column('flow_design_adjusted', sa.Numeric(), nullable=True), + sa.Column('flow_average', sa.Numeric(), nullable=True), + sa.Column('biosolids', sa.Numeric(), nullable=True), + sa.Column('excess_flow', sa.Numeric(), nullable=True), + sa.Column('biogas_utilized', sa.Boolean(), nullable=True), + sa.Column('flaring', sa.Boolean(), nullable=True), + sa.Column('excess_mass_loading_rate', sa.Numeric(), nullable=True), + sa.Column('excess_mass_loading_rate_wet', sa.Numeric(), nullable=True), + sa.Column('methane_production', sa.Numeric(), nullable=True), + sa.Column('energy_content', sa.Numeric(), nullable=True), + sa.Column('electric_kw', sa.Numeric(), nullable=True), + sa.Column('thermal_mmbtu_d', sa.Numeric(), nullable=True), + sa.Column('electric_kwh', sa.Numeric(), nullable=True), + sa.Column('thermal_annual_mmbtu_y', sa.Numeric(), nullable=True), + sa.Column('anaerobic_digestion_facility', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True), + sa.Column('dayload_bdt', sa.Numeric(), nullable=True), + sa.Column('dayload', sa.Numeric(), nullable=True), + sa.Column('equivalent_generation', sa.Numeric(), nullable=True), + sa.Column('facility_type', sa.Text(), nullable=True), + sa.Column('feedstock', sa.Text(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('latitude', sa.Numeric(), nullable=True), + sa.Column('longitude', sa.Numeric(), nullable=True), + sa.Column('zipcode', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('plant_id') + ) + op.create_table('lineage_group', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('location_resolution', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('lookup_base', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('method_abbrev', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('method_category', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('method_standard', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameter_category', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('polygon', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('geoid', sa.Text(), nullable=True), + sa.Column('geom', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('preparation_method_abbreviation', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('primary_ag_product', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('processing_method', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('provider', + sa.Column('codename', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_class', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_subclass', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('soil_type', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('source_type', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('strain', + sa.Column('parent_strain_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('unit', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_commodity', + sa.Column('usda_source', sa.Text(), nullable=True), + sa.Column('usda_code', sa.Text(), nullable=True), + sa.Column('parent_commodity_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['parent_commodity_id'], ['usda_commodity.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_domain', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_statistic_category', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_survey_program', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('data_source_type', + sa.Column('source_type_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['source_type_id'], ['source_type.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('location_address', + sa.Column('geography_id', sa.Text(), nullable=True), + sa.Column('address_line1', sa.Text(), nullable=True), + sa.Column('address_line2', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('zip', sa.Text(), nullable=True), + sa.Column('lat', sa.Float(), nullable=True), + sa.Column('lon', sa.Float(), nullable=True), + sa.Column('is_anonymous', sa.Boolean(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['geography_id'], ['geography.geoid'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameter', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('standard_unit_id', sa.Integer(), nullable=True), + sa.Column('calculated', sa.Boolean(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['standard_unit_id'], ['unit.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('preparation_method', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('prep_method_abbrev_id', sa.Integer(), nullable=True), + sa.Column('prep_temp_c', sa.Numeric(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.Column('drying_step', sa.Boolean(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['prep_method_abbrev_id'], ['preparation_method_abbreviation.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('primary_ag_product_id', sa.Integer(), nullable=True), + sa.Column('resource_class_id', sa.Integer(), nullable=True), + sa.Column('resource_subclass_id', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('test', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['primary_ag_product_id'], ['primary_ag_product.id'], ), + sa.ForeignKeyConstraint(['resource_class_id'], ['resource_class.id'], ), + sa.ForeignKeyConstraint(['resource_subclass_id'], ['resource_subclass.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_term_map', + sa.Column('source_system', sa.Text(), nullable=True), + sa.Column('source_context', sa.Text(), nullable=True), + sa.Column('raw_term', sa.Text(), nullable=True), + sa.Column('usda_commodity_id', sa.Integer(), nullable=True), + sa.Column('is_verified', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['usda_commodity_id'], ['usda_commodity.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('billion_ton2023_record', + sa.Column('subclass_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('geoid', sa.Text(), nullable=True), + sa.Column('county_square_miles', sa.Float(), nullable=True), + sa.Column('model_name', sa.Text(), nullable=True), + sa.Column('scenario_name', sa.Text(), nullable=True), + sa.Column('price_offered_usd', sa.Numeric(), nullable=True), + sa.Column('production', sa.Integer(), nullable=True), + sa.Column('production_unit_id', sa.Integer(), nullable=True), + sa.Column('btu_ton', sa.Integer(), nullable=True), + sa.Column('production_energy_content', sa.Integer(), nullable=True), + sa.Column('energy_content_unit_id', sa.Integer(), nullable=True), + sa.Column('product_density_dtpersqmi', sa.Numeric(), nullable=True), + sa.Column('land_source', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['energy_content_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['production_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.ForeignKeyConstraint(['subclass_id'], ['resource_subclass.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('data_source', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('data_source_type_id', sa.Integer(), nullable=True), + sa.Column('full_title', sa.Text(), nullable=True), + sa.Column('creator', sa.Text(), nullable=True), + sa.Column('subject', sa.Text(), nullable=True), + sa.Column('publisher', sa.Text(), nullable=True), + sa.Column('contributor', sa.Text(), nullable=True), + sa.Column('date', sa.DateTime(), nullable=True), + sa.Column('type', sa.Text(), nullable=True), + sa.Column('biocirv', sa.Boolean(), nullable=True), + sa.Column('format', sa.Text(), nullable=True), + sa.Column('language', sa.Text(), nullable=True), + sa.Column('relation', sa.Text(), nullable=True), + sa.Column('temporal_coverage', sa.Text(), nullable=True), + sa.Column('location_coverage_id', sa.Integer(), nullable=True), + sa.Column('rights', sa.Text(), nullable=True), + sa.Column('license', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['data_source_type_id'], ['data_source_type.id'], ), + sa.ForeignKeyConstraint(['location_coverage_id'], ['location_resolution.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('equipment', + sa.Column('equipment_location_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('uri', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['equipment_location_id'], ['location_address.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('experiment', + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('exper_start_date', sa.Date(), nullable=True), + sa.Column('exper_duration', sa.Numeric(), nullable=True), + sa.Column('exper_duration_unit_id', sa.Integer(), nullable=True), + sa.Column('exper_location_id', sa.Integer(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['exper_duration_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['exper_location_id'], ['location_address.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('field_sample', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('provider_id', sa.Integer(), nullable=True), + sa.Column('collector_id', sa.Integer(), nullable=True), + sa.Column('sample_collection_source', sa.Text(), nullable=True), + sa.Column('amount_collected', sa.Numeric(), nullable=True), + sa.Column('amount_collected_unit_id', sa.Integer(), nullable=True), + sa.Column('sampling_location_id', sa.Integer(), nullable=True), + sa.Column('field_storage_method_id', sa.Integer(), nullable=True), + sa.Column('field_storage_duration_value', sa.Numeric(), nullable=True), + sa.Column('field_storage_duration_unit_id', sa.Integer(), nullable=True), + sa.Column('field_storage_location_id', sa.Integer(), nullable=True), + sa.Column('collection_timestamp', sa.DateTime(), nullable=True), + sa.Column('collection_method_id', sa.Integer(), nullable=True), + sa.Column('harvest_method_id', sa.Integer(), nullable=True), + sa.Column('harvest_date', sa.Date(), nullable=True), + sa.Column('field_sample_storage_location_id', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['amount_collected_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['collection_method_id'], ['collection_method.id'], ), + sa.ForeignKeyConstraint(['collector_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['field_sample_storage_location_id'], ['location_address.id'], ), + sa.ForeignKeyConstraint(['field_storage_duration_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['field_storage_location_id'], ['location_address.id'], ), + sa.ForeignKeyConstraint(['field_storage_method_id'], ['field_storage_method.id'], ), + sa.ForeignKeyConstraint(['harvest_method_id'], ['harvest_method.id'], ), + sa.ForeignKeyConstraint(['provider_id'], ['provider.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.ForeignKeyConstraint(['sampling_location_id'], ['location_address.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('location_soil_type', + sa.Column('location_id', sa.Integer(), nullable=True), + sa.Column('soil_type_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['location_id'], ['location_address.id'], ), + sa.ForeignKeyConstraint(['soil_type_id'], ['soil_type.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameter_category_parameter', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('parameter_id', sa.Integer(), nullable=True), + sa.Column('parameter_category_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['parameter_category_id'], ['parameter_category.id'], ), + sa.ForeignKeyConstraint(['parameter_id'], ['parameter.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('parameter_unit', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('parameter_id', sa.Integer(), nullable=True), + sa.Column('alternate_unit_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['alternate_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['parameter_id'], ['parameter.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_availability', + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('geoid', sa.Text(), nullable=True), + sa.Column('from_month', sa.Integer(), nullable=True), + sa.Column('to_month', sa.Integer(), nullable=True), + sa.Column('year_round', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_morphology', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('morphology_uri', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_usda_commodity_map', + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('primary_ag_product_id', sa.Integer(), nullable=True), + sa.Column('usda_commodity_id', sa.Integer(), nullable=True), + sa.Column('match_tier', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['primary_ag_product_id'], ['primary_ag_product.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.ForeignKeyConstraint(['usda_commodity_id'], ['usda_commodity.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('dataset', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('record_type', sa.Text(), nullable=True), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('start_date', sa.Date(), nullable=True), + sa.Column('end_date', sa.Date(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['source_id'], ['data_source.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('experiment_analysis', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('analysis_type_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analysis_type_id'], ['analysis_type.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('experiment_equipment', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('equipment_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['equipment.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('facility_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('facility_name', sa.Text(), nullable=True), + sa.Column('location_id', sa.Integer(), nullable=True), + sa.Column('capacity_mw', sa.Numeric(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('operator', sa.Text(), nullable=True), + sa.Column('start_year', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['location_id'], ['location_address.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('field_sample_condition', + sa.Column('field_sample_id', sa.Integer(), nullable=True), + sa.Column('ag_treatment_id', sa.Integer(), nullable=True), + sa.Column('last_application_date', sa.Date(), nullable=True), + sa.Column('treatment_amount_per_acre', sa.Float(), nullable=True), + sa.Column('processing_method_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['ag_treatment_id'], ['ag_treatment.id'], ), + sa.ForeignKeyConstraint(['field_sample_id'], ['field_sample.id'], ), + sa.ForeignKeyConstraint(['processing_method_id'], ['processing_method.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file_object_metadata', + sa.Column('data_source_id', sa.Integer(), nullable=True), + sa.Column('bucket_path', sa.Text(), nullable=True), + sa.Column('file_format', sa.Text(), nullable=True), + sa.Column('file_size', sa.Integer(), nullable=True), + sa.Column('checksum_md5', sa.Text(), nullable=True), + sa.Column('checksum_sha256', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['data_source_id'], ['data_source.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('landiq_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('polygon_id', sa.Integer(), nullable=True), + sa.Column('main_crop', sa.Integer(), nullable=True), + sa.Column('secondary_crop', sa.Integer(), nullable=True), + sa.Column('tertiary_crop', sa.Integer(), nullable=True), + sa.Column('quaternary_crop', sa.Integer(), nullable=True), + sa.Column('confidence', sa.Integer(), nullable=True), + sa.Column('irrigated', sa.Boolean(), nullable=True), + sa.Column('acres', sa.Float(), nullable=True), + sa.Column('version', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['main_crop'], ['primary_ag_product.id'], ), + sa.ForeignKeyConstraint(['polygon_id'], ['polygon.id'], ), + sa.ForeignKeyConstraint(['quaternary_crop'], ['primary_ag_product.id'], ), + sa.ForeignKeyConstraint(['secondary_crop'], ['primary_ag_product.id'], ), + sa.ForeignKeyConstraint(['tertiary_crop'], ['primary_ag_product.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('method', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('method_abbrev_id', sa.Integer(), nullable=True), + sa.Column('method_category_id', sa.Integer(), nullable=True), + sa.Column('method_standard_id', sa.Integer(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('detection_limits', sa.Text(), nullable=True), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['method_abbrev_id'], ['method_abbrev.id'], ), + sa.ForeignKeyConstraint(['method_category_id'], ['method_category.id'], ), + sa.ForeignKeyConstraint(['method_standard_id'], ['method_standard.id'], ), + sa.ForeignKeyConstraint(['source_id'], ['data_source.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('observation', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('record_type', sa.Text(), nullable=True), + sa.Column('record_id', sa.Integer(), nullable=True), + sa.Column('parameter_id', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(), nullable=True), + sa.Column('unit_id', sa.Integer(), nullable=True), + sa.Column('dimension_type_id', sa.Integer(), nullable=True), + sa.Column('dimension_value', sa.Numeric(), nullable=True), + sa.Column('dimension_unit_id', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['dimension_type_id'], ['dimension_type.id'], ), + sa.ForeignKeyConstraint(['dimension_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['parameter_id'], ['parameter.id'], ), + sa.ForeignKeyConstraint(['unit_id'], ['unit.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('physical_characteristic', + sa.Column('field_sample_id', sa.Integer(), nullable=True), + sa.Column('particle_length', sa.Numeric(), nullable=True), + sa.Column('particle_width', sa.Numeric(), nullable=True), + sa.Column('particle_height', sa.Numeric(), nullable=True), + sa.Column('particle_unit_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['field_sample_id'], ['field_sample.id'], ), + sa.ForeignKeyConstraint(['particle_unit_id'], ['unit.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('prepared_sample', + sa.Column('name', sa.Text(), nullable=True), + sa.Column('field_sample_id', sa.Integer(), nullable=True), + sa.Column('prep_method_id', sa.Integer(), nullable=True), + sa.Column('prep_date', sa.Date(), nullable=True), + sa.Column('preparer_id', sa.Integer(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['field_sample_id'], ['field_sample.id'], ), + sa.ForeignKeyConstraint(['prep_method_id'], ['preparation_method.id'], ), + sa.ForeignKeyConstraint(['preparer_id'], ['contact.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_census_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('geoid', sa.Text(), nullable=True), + sa.Column('commodity_code', sa.Integer(), nullable=True), + sa.Column('year', sa.Integer(), nullable=True), + sa.Column('source_reference', sa.Text(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['commodity_code'], ['usda_commodity.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_market_report', + sa.Column('slug_id', sa.Integer(), nullable=True), + sa.Column('slug_name', sa.Text(), nullable=True), + sa.Column('report_series_title', sa.Text(), nullable=True), + sa.Column('frequency', sa.Text(), nullable=True), + sa.Column('office_name', sa.Text(), nullable=True), + sa.Column('office_city_id', sa.Integer(), nullable=True), + sa.Column('office_state_fips', sa.Text(), nullable=True), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['office_city_id'], ['location_address.id'], ), + sa.ForeignKeyConstraint(['source_id'], ['data_source.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_survey_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('geoid', sa.Text(), nullable=True), + sa.Column('commodity_code', sa.Integer(), nullable=True), + sa.Column('year', sa.Integer(), nullable=True), + sa.Column('survey_program_id', sa.Integer(), nullable=True), + sa.Column('survey_period', sa.Text(), nullable=True), + sa.Column('reference_month', sa.Text(), nullable=True), + sa.Column('seasonal_flag', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['commodity_code'], ['usda_commodity.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['survey_program_id'], ['usda_survey_program.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('aim1_record_base', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('aim2_record_base', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('calc_method_id', sa.Integer(), nullable=True), + sa.Column('calc_analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('autoclave_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('calc_method_id', sa.Integer(), nullable=True), + sa.Column('calc_analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('calorimetry_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('compositional_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('experiment_method', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('experiment_prepared_sample', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('prepared_sample_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['prepared_sample_id'], ['prepared_sample.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fermentation_record', + sa.Column('strain_id', sa.Integer(), nullable=True), + sa.Column('pretreatment_method_id', sa.Integer(), nullable=True), + sa.Column('eh_method_id', sa.Integer(), nullable=True), + sa.Column('well_position', sa.Text(), nullable=True), + sa.Column('temperature', sa.Numeric(), nullable=True), + sa.Column('agitation_rpm', sa.Numeric(), nullable=True), + sa.Column('vessel_id', sa.Integer(), nullable=True), + sa.Column('analyte_detection_equipment_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('calc_method_id', sa.Integer(), nullable=True), + sa.Column('calc_analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ftnir_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('gasification_record', + sa.Column('feedstock_mass', sa.Numeric(), nullable=True), + sa.Column('bed_temperature', sa.Numeric(), nullable=True), + sa.Column('gas_flow_rate', sa.Numeric(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('calc_method_id', sa.Integer(), nullable=True), + sa.Column('calc_analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('icp_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('pretreatment_record', + sa.Column('pretreatment_method_id', sa.Integer(), nullable=True), + sa.Column('eh_method_id', sa.Integer(), nullable=True), + sa.Column('reaction_block_id', sa.Integer(), nullable=True), + sa.Column('block_position', sa.Text(), nullable=True), + sa.Column('temperature', sa.Numeric(), nullable=True), + sa.Column('replicate_no', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('calc_method_id', sa.Integer(), nullable=True), + sa.Column('calc_analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('proximate_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('rgb_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ultimate_record', + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('usda_market_record', + sa.Column('report_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('report_begin_date', sa.DateTime(), nullable=True), + sa.Column('report_end_date', sa.DateTime(), nullable=True), + sa.Column('report_date', sa.DateTime(), nullable=True), + sa.Column('commodity_id', sa.Integer(), nullable=True), + sa.Column('market_type_id', sa.Integer(), nullable=True), + sa.Column('market_type_category', sa.Text(), nullable=True), + sa.Column('grp', sa.Text(), nullable=True), + sa.Column('market_category_id', sa.Integer(), nullable=True), + sa.Column('class_', sa.Text(), nullable=True), + sa.Column('grade', sa.Text(), nullable=True), + sa.Column('variety', sa.Text(), nullable=True), + sa.Column('protein_pct', sa.Numeric(), nullable=True), + sa.Column('application', sa.Text(), nullable=True), + sa.Column('pkg', sa.Text(), nullable=True), + sa.Column('sale_type', sa.Text(), nullable=True), + sa.Column('price_unit_id', sa.Integer(), nullable=True), + sa.Column('freight', sa.Text(), nullable=True), + sa.Column('trans_mode', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['commodity_id'], ['usda_commodity.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['price_unit_id'], ['unit.id'], ), + sa.ForeignKeyConstraint(['report_id'], ['usda_market_report.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('xrd_record', + sa.Column('scan_low_nm', sa.Integer(), nullable=True), + sa.Column('scan_high_nm', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('xrf_record', + sa.Column('wavelength_nm', sa.Numeric(), nullable=True), + sa.Column('intensity', sa.Numeric(), nullable=True), + sa.Column('energy_slope', sa.Numeric(), nullable=True), + sa.Column('energy_offset', sa.Numeric(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('experiment_id', sa.Integer(), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('sample_id', sa.Integer(), nullable=True), + sa.Column('technical_replicate_no', sa.Integer(), nullable=True), + sa.Column('technical_replicate_total', sa.Integer(), nullable=True), + sa.Column('method_id', sa.Integer(), nullable=True), + sa.Column('analyst_id', sa.Integer(), nullable=True), + sa.Column('raw_data_id', sa.Integer(), nullable=True), + sa.Column('qc_pass', sa.Boolean(), nullable=True), + sa.Column('note', sa.Text(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('etl_run_id', sa.Text(), nullable=True), + sa.Column('lineage_group_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['analyst_id'], ['contact.id'], ), + sa.ForeignKeyConstraint(['dataset_id'], ['data_source.id'], ), + sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ), + sa.ForeignKeyConstraint(['method_id'], ['method.id'], ), + sa.ForeignKeyConstraint(['raw_data_id'], ['file_object_metadata.id'], ), + sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('xrf_record') + op.drop_table('xrd_record') + op.drop_table('usda_market_record') + op.drop_table('ultimate_record') + op.drop_table('rgb_record') + op.drop_table('proximate_record') + op.drop_table('pretreatment_record') + op.drop_table('icp_record') + op.drop_table('gasification_record') + op.drop_table('ftnir_record') + op.drop_table('fermentation_record') + op.drop_table('experiment_prepared_sample') + op.drop_table('experiment_method') + op.drop_table('compositional_record') + op.drop_table('calorimetry_record') + op.drop_table('autoclave_record') + op.drop_table('aim2_record_base') + op.drop_table('aim1_record_base') + op.drop_table('usda_survey_record') + op.drop_table('usda_market_report') + op.drop_table('usda_census_record') + op.drop_table('prepared_sample') + op.drop_table('physical_characteristic') + op.drop_table('observation') + op.drop_table('method') + op.drop_table('landiq_record') + op.drop_table('file_object_metadata') + op.drop_table('field_sample_condition') + op.drop_table('facility_record') + op.drop_table('experiment_equipment') + op.drop_table('experiment_analysis') + op.drop_table('dataset') + op.drop_table('resource_usda_commodity_map') + op.drop_table('resource_morphology') + op.drop_table('resource_availability') + op.drop_table('parameter_unit') + op.drop_table('parameter_category_parameter') + op.drop_table('location_soil_type') + op.drop_table('field_sample') + op.drop_table('experiment') + op.drop_table('equipment') + op.drop_table('data_source') + op.drop_table('billion_ton2023_record') + op.drop_table('usda_term_map') + op.drop_table('resource') + op.drop_table('preparation_method') + op.drop_table('parameter') + op.drop_table('location_address') + op.drop_table('data_source_type') + op.drop_table('usda_survey_program') + op.drop_table('usda_statistic_category') + op.drop_table('usda_domain') + op.drop_table('usda_commodity') + op.drop_table('unit') + op.drop_table('strain') + op.drop_table('source_type') + op.drop_table('soil_type') + op.drop_table('resource_subclass') + op.drop_table('resource_class') + op.drop_table('provider') + op.drop_table('processing_method') + op.drop_table('primary_ag_product') + op.drop_table('preparation_method_abbreviation') + op.drop_table('polygon') + op.drop_table('parameter_category') + op.drop_table('method_standard') + op.drop_table('method_category') + op.drop_table('method_abbrev') + op.drop_table('lookup_base') + op.drop_table('location_resolution') + op.drop_table('lineage_group') + op.drop_table('infrastructure_wastewater_treatment_plants') + op.drop_table('infrastructure_saf_and_renewable_diesel_plants') + op.drop_table('infrastructure_msw_to_energy_anaerobic_digesters') + op.drop_table('infrastructure_livestock_anaerobic_digesters') + op.drop_table('infrastructure_landfills') + op.drop_table('infrastructure_food_processing_facilities') + op.drop_table('infrastructure_ethanol_biorefineries') + op.drop_table('infrastructure_district_energy_systems') + op.drop_table('infrastructure_combustion_plants') + op.drop_table('infrastructure_cafo_manure_locations') + op.drop_table('infrastructure_biosolids_facilities') + op.drop_table('infrastructure_biodiesel_plants') + op.drop_table('harvest_method') + op.drop_table('geography') + op.drop_table('field_storage_method') + op.drop_table('etl_run') + op.drop_table('entity_lineage') + op.drop_table('dimension_type') + op.drop_table('contact') + op.drop_table('collection_method') + op.drop_table('base_entity') + op.drop_table('analysis_type') + op.drop_table('ag_treatment') + # ### end Alembic commands ### diff --git a/alembic/versions/1ace3fc9c593_initial_migration.py b/alembic/versions/1ace3fc9c593_initial_migration.py new file mode 100644 index 00000000..de2d3da7 --- /dev/null +++ b/alembic/versions/1ace3fc9c593_initial_migration.py @@ -0,0 +1,32 @@ +"""Initial migration + +Revision ID: 1ace3fc9c593 +Revises: 62ef89b0050a +Create Date: 2026-01-07 18:39:13.836533 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1ace3fc9c593' +down_revision: Union[str, Sequence[str], None] = '62ef89b0050a' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/2b18e6e5e72d_trying_with_fixing_the_base.py b/alembic/versions/2b18e6e5e72d_trying_with_fixing_the_base.py new file mode 100644 index 00000000..bf17d4ad --- /dev/null +++ b/alembic/versions/2b18e6e5e72d_trying_with_fixing_the_base.py @@ -0,0 +1,32 @@ +"""trying with fixing the base + +Revision ID: 2b18e6e5e72d +Revises: 1ace3fc9c593 +Create Date: 2026-01-07 20:55:34.214689 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2b18e6e5e72d' +down_revision: Union[str, Sequence[str], None] = '1ace3fc9c593' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/2b41e390fc5d_fixed_alembic_environment_to_import_.py b/alembic/versions/2b41e390fc5d_fixed_alembic_environment_to_import_.py new file mode 100644 index 00000000..ca65e62d --- /dev/null +++ b/alembic/versions/2b41e390fc5d_fixed_alembic_environment_to_import_.py @@ -0,0 +1,32 @@ +"""Fixed alembic environment to import generated schemas again + +Revision ID: 2b41e390fc5d +Revises: a8a3ea954e88 +Create Date: 2026-01-07 18:18:18.527886 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2b41e390fc5d' +down_revision: Union[str, Sequence[str], None] = 'a8a3ea954e88' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/62ef89b0050a_fixed_alembic_gen_sqla_so_all_inherit_.py b/alembic/versions/62ef89b0050a_fixed_alembic_gen_sqla_so_all_inherit_.py new file mode 100644 index 00000000..80ad61d8 --- /dev/null +++ b/alembic/versions/62ef89b0050a_fixed_alembic_gen_sqla_so_all_inherit_.py @@ -0,0 +1,32 @@ +"""Fixed alembic gen_sqla so all inherit from same base + +Revision ID: 62ef89b0050a +Revises: 2b41e390fc5d +Create Date: 2026-01-07 18:33:17.030868 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '62ef89b0050a' +down_revision: Union[str, Sequence[str], None] = '2b41e390fc5d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/9be24ffa7bb9_refactor_and_update_env_py.py b/alembic/versions/9be24ffa7bb9_refactor_and_update_env_py.py new file mode 100644 index 00000000..114405dc --- /dev/null +++ b/alembic/versions/9be24ffa7bb9_refactor_and_update_env_py.py @@ -0,0 +1,1462 @@ +"""refactor and update env.py + +Revision ID: 9be24ffa7bb9 +Revises: 15467b7f2f3d +Create Date: 2026-01-07 16:50:27.201523 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '9be24ffa7bb9' +down_revision: Union[str, Sequence[str], None] = '15467b7f2f3d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('resource_subclass') + op.drop_table('usda_market_report') + op.drop_table('infrastructure_cafo_manure_locations') + op.drop_table('experiment_equipment') + op.drop_table('lookup_base') + op.drop_table('infrastructure_ethanol_biorefineries') + op.drop_table('etl_run') + op.drop_table('polygon') + op.drop_table('calorimetry_record') + op.drop_table('base_entity') + op.drop_table('method') + op.drop_table('infrastructure_landfills') + op.drop_table('observation') + op.drop_table('equipment') + op.drop_table('pretreatment_record') + op.drop_table('infrastructure_biosolids_facilities') + op.drop_table('prepared_sample') + op.drop_table('infrastructure_district_energy_systems') + op.drop_table('parameter_category') + op.drop_table('xrd_record') + op.drop_table('resource_class') + op.drop_table('preparation_method') + op.drop_table('strain') + op.drop_table('infrastructure_food_processing_facilities') + op.drop_table('file_object_metadata') + op.drop_table('ftnir_record') + op.drop_table('collection_method') + op.drop_table('harvest_method') + op.drop_table('physical_characteristic') + op.drop_table('experiment') + op.drop_table('parameter') + op.drop_table('infrastructure_livestock_anaerobic_digesters') + op.drop_table('experiment_analysis') + op.drop_table('preparation_method_abbreviation') + op.drop_table('usda_term_map') + op.drop_table('xrf_record') + op.drop_table('source_type') + op.drop_table('lineage_group') + op.drop_table('icp_record') + op.drop_table('infrastructure_wastewater_treatment_plants') + op.drop_table('usda_commodity') + op.drop_table('infrastructure_combustion_plants') + op.drop_table('aim1_record_base') + op.drop_table('ultimate_record') + op.drop_table('soil_type') + op.drop_table('usda_survey_record') + op.drop_table('facility_record') + op.drop_table('autoclave_record') + op.drop_table('usda_domain') + op.drop_table('gasification_record') + op.drop_table('infrastructure_msw_to_energy_anaerobic_digesters') + op.drop_table('ag_treatment') + op.drop_table('location_resolution') + op.drop_table('analysis_type') + op.drop_table('method_category') + op.drop_table('resource_morphology') + op.drop_table('rgb_record') + op.drop_table('primary_ag_product') + op.drop_table('dimension_type') + op.drop_table('provider') + op.drop_table('location_address') + op.drop_table('parameter_category_parameter') + op.drop_table('resource_availability') + op.drop_table('billion_ton2023_record') + op.drop_table('processing_method') + op.drop_table('parameter_unit') + op.drop_table('infrastructure_saf_and_renewable_diesel_plants') + op.drop_table('usda_survey_program') + op.drop_table('method_abbrev') + op.drop_table('infrastructure_biodiesel_plants') + op.drop_table('compositional_record') + op.drop_table('resource_usda_commodity_map') + op.drop_table('dataset') + op.drop_table('field_sample') + op.drop_table('field_sample_condition') + op.drop_table('data_source') + op.drop_table('landiq_record') + op.drop_table('field_storage_method') + op.drop_table('method_standard') + op.drop_table('experiment_method') + op.drop_table('usda_census_record') + op.drop_table('location_soil_type') + op.drop_table('experiment_prepared_sample') + op.drop_table('fermentation_record') + op.drop_table('entity_lineage') + op.drop_table('data_source_type') + op.drop_table('resource_counterfactual') + op.drop_table('contact') + op.drop_table('proximate_record') + op.drop_table('usda_statistic_category') + op.drop_table('unit') + op.drop_table('aim2_record_base') + op.drop_table('geography') + op.drop_table('resource') + op.drop_table('usda_market_record') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('usda_market_record', + sa.Column('report_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('report_begin_date', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('report_end_date', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('report_date', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('commodity_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('market_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('grp', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('class_', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('grade', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('variety', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('protein_pct', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('application', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('pkg', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('sale_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('price_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('freight', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('trans_mode', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('market_type_category', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('market_category_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_market_record_pkey')) + ) + op.create_table('resource', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('resource_class_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_subclass_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('test', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('primary_ag_product_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_pkey')) + ) + op.create_table('geography', + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('state_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state_fips', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county_fips', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('region_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('agg_level_desc', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('geoid', name=op.f('geography_pkey')) + ) + op.create_table('aim2_record_base', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('aim2_record_base_pkey')) + ) + op.create_table('unit', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('unit_pkey')) + ) + op.create_table('usda_statistic_category', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_statistic_category_pkey')) + ) + op.create_table('proximate_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('proximate_record_pkey')) + ) + op.create_table('contact', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('first_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('last_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('email', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('affiliation', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('contact_pkey')) + ) + op.create_table('resource_counterfactual', + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('counterfactual_description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('animal_bedding_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('animal_bedding_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('animal_feed_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('animal_feed_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('bioelectricty_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('bioelectricty_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('burn_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('burn_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('compost_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('compost_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('landfill_percent', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('landfill_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('counterfactual_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_counterfactual_pkey')) + ) + op.create_table('data_source_type', + sa.Column('source_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('data_source_type_pkey')) + ) + op.create_table('entity_lineage', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('source_table', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source_row_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('entity_lineage_pkey')) + ) + op.create_table('fermentation_record', + sa.Column('strain_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('pretreatment_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('eh_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('well_position', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('temperature', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('agitation_rpm', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('vessel_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyte_detection_equipment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('fermentation_record_pkey')) + ) + op.create_table('experiment_prepared_sample', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('prepared_sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('experiment_prepared_sample_pkey')) + ) + op.create_table('location_soil_type', + sa.Column('location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('soil_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('location_soil_type_pkey')) + ) + op.create_table('usda_census_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('commodity_code', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('year', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('source_reference', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_census_record_pkey')) + ) + op.create_table('experiment_method', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('experiment_method_pkey')) + ) + op.create_table('method_standard', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('method_standard_pkey')) + ) + op.create_table('field_storage_method', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('field_storage_method_pkey')) + ) + op.create_table('landiq_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('polygon_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('main_crop', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('secondary_crop', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('tertiary_crop', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('quaternary_crop', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('confidence', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('irrigated', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('acres', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), + sa.Column('version', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('landiq_record_pkey')) + ) + op.create_table('data_source', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('publisher', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('license', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('data_source_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('full_title', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('creator', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('subject', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('contributor', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('date', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biocirv', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('format', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('language', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('relation', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('temporal_coverage', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('location_coverage_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('rights', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('data_source_pkey')) + ) + op.create_table('field_sample_condition', + sa.Column('field_sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('ag_treatment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('last_application_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('treatment_amount_per_acre', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), + sa.Column('processing_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('field_sample_condition_pkey')) + ) + op.create_table('field_sample', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('provider_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('collector_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_collection_source', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('amount_collected', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('amount_collected_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sampling_location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('field_storage_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('field_storage_duration_value', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('field_storage_duration_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('field_storage_location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('collection_timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('collection_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('harvest_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('harvest_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('field_sample_storage_location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('field_sample_pkey')) + ) + op.create_table('dataset', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('record_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('start_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('end_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('dataset_pkey')) + ) + op.create_table('resource_usda_commodity_map', + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('usda_commodity_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('match_tier', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('primary_ag_product_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_usda_commodity_map_pkey')) + ) + op.create_table('compositional_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('compositional_record_pkey')) + ) + op.create_table('infrastructure_biodiesel_plants', + sa.Column('biodiesel_plant_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('company', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('bbi_index', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('capacity_mmg_per_y', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('feedstock', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('coordinates', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('source', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('biodiesel_plant_id', name=op.f('infrastructure_biodiesel_plants_pkey')) + ) + op.create_table('method_abbrev', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('method_abbrev_pkey')) + ) + op.create_table('usda_survey_program', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_survey_program_pkey')) + ) + op.create_table('infrastructure_saf_and_renewable_diesel_plants', + sa.Column('ibcc_index', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('company', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('country', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('capacity', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('feedstock', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('products', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('coordinates', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('ibcc_index', name=op.f('infrastructure_saf_and_renewable_diesel_plants_pkey')) + ) + op.create_table('parameter_unit', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('parameter_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('alternate_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('parameter_unit_pkey')) + ) + op.create_table('processing_method', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('processing_method_pkey')) + ) + op.create_table('billion_ton2023_record', + sa.Column('subclass_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county_square_miles', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), + sa.Column('model_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('scenario_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('price_offered_usd', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('production', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('production_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('btu_ton', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('production_energy_content', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('energy_content_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('product_density_dtpersqmi', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('land_source', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('billion_ton2023_record_pkey')) + ) + op.create_table('resource_availability', + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('from_month', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('to_month', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('year_round', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_availability_pkey')) + ) + op.create_table('parameter_category_parameter', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('parameter_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('parameter_category_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('parameter_category_parameter_pkey')) + ) + op.create_table('location_address', + sa.Column('geography_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address_line1', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address_line2', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('zip', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lat', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), + sa.Column('lon', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), + sa.Column('is_anonymous', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('location_address_pkey')) + ) + op.create_table('provider', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('codename', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('provider_pkey')) + ) + op.create_table('dimension_type', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('dimension_type_pkey')) + ) + op.create_table('primary_ag_product', + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('primary_ag_product_pkey')) + ) + op.create_table('rgb_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('rgb_record_pkey')) + ) + op.create_table('resource_morphology', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('morphology_uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_morphology_pkey')) + ) + op.create_table('method_category', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('method_category_pkey')) + ) + op.create_table('analysis_type', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('analysis_type_pkey')) + ) + op.create_table('location_resolution', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('location_resolution_pkey')) + ) + op.create_table('ag_treatment', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('ag_treatment_pkey')) + ) + op.create_table('infrastructure_msw_to_energy_anaerobic_digesters', + sa.Column('wte_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('equivalent_generation', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('feedstock', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('dayload', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('dayload_bdt', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('facility_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('wkt_geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('wte_id', name=op.f('infrastructure_msw_to_energy_anaerobic_digesters_pkey')) + ) + op.create_table('gasification_record', + sa.Column('feedstock_mass', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('bed_temperature', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('gas_flow_rate', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('gasification_record_pkey')) + ) + op.create_table('usda_domain', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_domain_pkey')) + ) + op.create_table('autoclave_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('autoclave_record_pkey')) + ) + op.create_table('facility_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('facility_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('capacity_mw', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('operator', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('start_year', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('facility_record_pkey')) + ) + op.create_table('usda_survey_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('commodity_code', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('year', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('survey_program_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('survey_period', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('reference_month', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('seasonal_flag', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_survey_record_pkey')) + ) + op.create_table('soil_type', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('soil_type_pkey')) + ) + op.create_table('ultimate_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('ultimate_record_pkey')) + ) + op.create_table('aim1_record_base', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('aim1_record_base_pkey')) + ) + op.create_table('infrastructure_combustion_plants', + sa.Column('combustion_fid', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('objectid', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('equivalent_generation', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('np_mw', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('cf', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('yearload', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('fuel', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('wkt_geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('combustion_fid', name=op.f('infrastructure_combustion_plants_pkey')) + ) + op.create_table('usda_commodity', + sa.Column('usda_source', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('usda_code', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('parent_commodity_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_commodity_pkey')) + ) + op.create_table('infrastructure_wastewater_treatment_plants', + sa.Column('plant_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('codigestion', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('flow_design_adjusted', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('flow_average', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('biosolids', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('excess_flow', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('biogas_utilized', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('flaring', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('excess_mass_loading_rate', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('excess_mass_loading_rate_wet', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('methane_production', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('energy_content', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('electric_kw', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('thermal_mmbtu_d', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('electric_kwh', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('thermal_annual_mmbtu_y', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('anaerobic_digestion_facility', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('dayload_bdt', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('dayload', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('equivalent_generation', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('facility_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('feedstock', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('zipcode', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('plant_id', name=op.f('infrastructure_wastewater_treatment_plants_pkey')) + ) + op.create_table('icp_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('icp_record_pkey')) + ) + op.create_table('lineage_group', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('lineage_group_pkey')) + ) + op.create_table('source_type', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('source_type_pkey')) + ) + op.create_table('xrf_record', + sa.Column('maybe_wavelength_nm', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('maybe_intensity', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('maybe_energy_slope', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('maybe_energy_offset', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('xrf_record_pkey')) + ) + op.create_table('usda_term_map', + sa.Column('source_system', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source_context', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('raw_term', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('usda_commodity_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('is_verified', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_term_map_pkey')) + ) + op.create_table('preparation_method_abbreviation', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('preparation_method_abbreviation_pkey')) + ) + op.create_table('experiment_analysis', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analysis_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('experiment_analysis_pkey')) + ) + op.create_table('infrastructure_livestock_anaerobic_digesters', + sa.Column('digester_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('project_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('digester_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('profile', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('year_operational', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('animal_type_class', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('animal_types', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('pop_feeding_digester', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('total_pop_feeding_digester', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('cattle', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dairy', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('poultry', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('swine', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('codigestion', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biogas_generation_estimate', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('electricity_generated', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('biogas_end_uses', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('methane_emission_reductions', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('digester_id', name=op.f('infrastructure_livestock_anaerobic_digesters_pkey')) + ) + op.create_table('parameter', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('standard_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calculated', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('parameter_pkey')) + ) + op.create_table('experiment', + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('exper_start_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('exper_duration', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('exper_duration_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('exper_location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('experiment_pkey')) + ) + op.create_table('physical_characteristic', + sa.Column('field_sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('particle_length', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('particle_width', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('particle_height', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('particle_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('physical_characteristic_pkey')) + ) + op.create_table('harvest_method', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('harvest_method_pkey')) + ) + op.create_table('collection_method', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('collection_method_pkey')) + ) + op.create_table('ftnir_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('ftnir_record_pkey')) + ) + op.create_table('file_object_metadata', + sa.Column('data_source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('bucket_path', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('file_format', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('file_size', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('checksum_md5', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('checksum_sha256', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('file_object_metadata_pkey')) + ) + op.create_table('infrastructure_food_processing_facilities', + sa.Column('processing_facility_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('company', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('join_count', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('master_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('subtype', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('target_fid', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('processing_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('zip', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('wkt_geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('processing_facility_id', name=op.f('infrastructure_food_processing_facilities_pkey')) + ) + op.create_table('strain', + sa.Column('parent_strain_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('strain_pkey')) + ) + op.create_table('preparation_method', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('prep_method_abbrev_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('prep_temp_c', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('drying_step', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('preparation_method_pkey')) + ) + op.create_table('resource_class', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_class_pkey')) + ) + op.create_table('xrd_record', + sa.Column('maybe_scan_low_nm', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('maybe_scan_high_nm', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('xrd_record_pkey')) + ) + op.create_table('parameter_category', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('parameter_category_pkey')) + ) + op.create_table('infrastructure_district_energy_systems', + sa.Column('des_fid', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('cbg_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('system', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('object_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('primary_fuel', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('secondary_fuel', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('usetype', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('cap_st', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('cap_hw', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('cap_cw', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('chpcg_cap', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('excess_c', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('excess_h', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('wkt_geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('des_fid', name=op.f('infrastructure_district_energy_systems_pkey')) + ) + op.create_table('prepared_sample', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('field_sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('prep_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('prep_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('preparer_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('prepared_sample_pkey')) + ) + op.create_table('infrastructure_biosolids_facilities', + sa.Column('biosolid_facility_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('report_submitted_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('facility', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('authority', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('plant_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('aqmd', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('facility_address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('facility_city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('facility_zip', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('facility_county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('mailing_street_1', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('mailing_city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('mailing_state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('mailing_zip', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biosolids_number', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biosolids_contact', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biosolids_contact_phone', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('biosolids_contact_email', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('adwf', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('potw_biosolids_generated', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('twtds_biosolids_treated', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('class_b_land_app', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('class_b_applier', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('class_a_compost', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('class_a_heat_dried', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('class_a_other', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('class_a_other_applier', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('twtds_transfer_to_second_preparer', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('twtds_second_preparer_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('adc_or_final_c', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('landfill', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('landfill_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('surface_disposal', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('deepwell_injection', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('stored', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('longterm_treatment', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('other', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('name_of_other', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('incineration', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('biosolid_facility_id', name=op.f('infrastructure_biosolids_facilities_pkey')) + ) + op.create_table('pretreatment_record', + sa.Column('pretreatment_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('eh_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('reaction_block_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('block_position', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('temperature', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('calc_analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pretreatment_record_pkey')) + ) + op.create_table('equipment', + sa.Column('equipment_location_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('equipment_pkey')) + ) + op.create_table('observation', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('record_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('record_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('parameter_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('value', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dimension_type_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('dimension_value', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('dimension_unit_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('observation_pkey')) + ) + op.create_table('infrastructure_landfills', + sa.Column('project_id', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('project_int_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('ghgrp_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('landfill_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('landfill_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('physical_address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('county', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('zip_code', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('ownership_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('landfill_owner_orgs', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('landfill_opened_year', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('landfill_closure_year', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('landfill_status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('waste_in_place', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('waste_in_place_year', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('lfg_system_in_place', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('lfg_collected', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('lfg_flared', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('project_status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('project_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('project_start_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('project_shutdown_date', sa.DATE(), autoincrement=False, nullable=True), + sa.Column('project_type_category', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lfg_energy_project_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('rng_delivery_method', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('actual_mw_generation', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('rated_mw_capacity', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('lfg_flow_to_project', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('direct_emission_reductions', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('avoided_emission_reductions', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('project_id', name=op.f('infrastructure_landfills_pkey')) + ) + op.create_table('method', + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('method_abbrev_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_category_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_standard_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('detection_limits', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('method_pkey')) + ) + op.create_table('base_entity', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('base_entity_pkey')) + ) + op.create_table('calorimetry_record', + sa.Column('dataset_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('sample_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_no', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('technical_replicate_total', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('method_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('raw_data_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('qc_pass', sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('analyst_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('calorimetry_record_pkey')) + ) + op.create_table('polygon', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('geoid', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('geom', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('polygon_pkey')) + ) + op.create_table('etl_run', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('started_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('completed_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('pipeline_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('status', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('records_ingested', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('note', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('etl_run_pkey')) + ) + op.create_table('infrastructure_ethanol_biorefineries', + sa.Column('ethanol_biorefinery_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('city', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('capacity_mgy', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('production_mgy', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('constr_exp', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('ethanol_biorefinery_id', name=op.f('infrastructure_ethanol_biorefineries_pkey')) + ) + op.create_table('lookup_base', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('lookup_base_pkey')) + ) + op.create_table('experiment_equipment', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('experiment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('equipment_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('experiment_equipment_pkey')) + ) + op.create_table('infrastructure_cafo_manure_locations', + sa.Column('cafo_manure_id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('latitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('longitude', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('owner_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('facility_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('address', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('town', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('state', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('zip', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('animal', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('animal_feed_operation_type', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('animal_units', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('animal_count', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('manure_total_solids', sa.NUMERIC(), autoincrement=False, nullable=True), + sa.Column('source', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('date_accessed', sa.DATE(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('cafo_manure_id', name=op.f('infrastructure_cafo_manure_locations_pkey')) + ) + op.create_table('usda_market_report', + sa.Column('slug_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('slug_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('report_series_title', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('frequency', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('office_name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('office_city_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('office_state_fips', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('source_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('etl_run_id', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('lineage_group_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('usda_market_report_pkey')) + ) + op.create_table('resource_subclass', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('name', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('uri', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('resource_subclass_pkey')) + ) + # ### end Alembic commands ### diff --git a/alembic/versions/a8a3ea954e88_fixed_alembic_environment_to_import_.py b/alembic/versions/a8a3ea954e88_fixed_alembic_environment_to_import_.py new file mode 100644 index 00000000..dbf56278 --- /dev/null +++ b/alembic/versions/a8a3ea954e88_fixed_alembic_environment_to_import_.py @@ -0,0 +1,32 @@ +"""Fixed alembic environment to import generated schemas + +Revision ID: a8a3ea954e88 +Revises: 9be24ffa7bb9 +Create Date: 2026-01-07 17:03:49.308268 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'a8a3ea954e88' +down_revision: Union[str, Sequence[str], None] = '9be24ffa7bb9' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/database.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/database.py index 00dc9b3c..058d1c98 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/database.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/database.py @@ -1,23 +1,25 @@ -from sqlmodel import SQLModel, create_engine, Session +from sqlalchemy import create_engine +from sqlalchemy.orm import declarative_base, sessionmaker from .config import settings # Create engine using the database URL from settings # echo=False by default to avoid verbose SQL logging engine = create_engine(settings.database_url, echo=False) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() + def get_session(): """ Dependency that yields a database session. Useful for FastAPI dependencies. """ - with Session(engine) as session: - yield session - -def create_db_and_tables(): - """ - Create all tables defined in SQLModel metadata. - """ - SQLModel.metadata.create_all(engine) + db = SessionLocal() + try: + yield db + finally: + db.close() def get_engine(): """ diff --git a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py index 0fcd6b01..9d5955cd 100644 --- a/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py +++ b/src/ca_biositing/datamodels/ca_biositing/datamodels/schemas/generated/ca_biositing.py @@ -1,11 +1,13 @@ +from ...database import Base + +from ...database import Base + from sqlalchemy import Column, Index, Table, ForeignKey from sqlalchemy.orm import relationship from sqlalchemy.sql.sqltypes import * -from sqlalchemy.orm import declarative_base from sqlalchemy.ext.associationproxy import association_proxy -Base = declarative_base() metadata = Base.metadata @@ -57,8 +59,8 @@ class ExperimentAnalysis(Base): __tablename__ = 'experiment_analysis' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - analysis_type_id = Column(Integer(), ForeignKey('AnalysisType.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + analysis_type_id = Column(Integer(), ForeignKey('analysis_type.id')) def __repr__(self): @@ -76,8 +78,8 @@ class ExperimentEquipment(Base): __tablename__ = 'experiment_equipment' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - equipment_id = Column(Integer(), ForeignKey('Equipment.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + equipment_id = Column(Integer(), ForeignKey('equipment.id')) def __repr__(self): @@ -95,8 +97,8 @@ class ExperimentMethod(Base): __tablename__ = 'experiment_method' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - method_id = Column(Integer(), ForeignKey('Method.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + method_id = Column(Integer(), ForeignKey('method.id')) def __repr__(self): @@ -114,8 +116,8 @@ class ExperimentPreparedSample(Base): __tablename__ = 'experiment_prepared_sample' id = Column(Integer(), primary_key=True, nullable=False ) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - prepared_sample_id = Column(Integer(), ForeignKey('PreparedSample.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + prepared_sample_id = Column(Integer(), ForeignKey('prepared_sample.id')) def __repr__(self): @@ -681,8 +683,8 @@ class ParameterCategoryParameter(Base): __tablename__ = 'parameter_category_parameter' id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer(), ForeignKey('Parameter.id')) - parameter_category_id = Column(Integer(), ForeignKey('ParameterCategory.id')) + parameter_id = Column(Integer(), ForeignKey('parameter.id')) + parameter_category_id = Column(Integer(), ForeignKey('parameter_category.id')) def __repr__(self): @@ -700,8 +702,8 @@ class ParameterUnit(Base): __tablename__ = 'parameter_unit' id = Column(Integer(), primary_key=True, nullable=False ) - parameter_id = Column(Integer(), ForeignKey('Parameter.id')) - alternate_unit_id = Column(Integer(), ForeignKey('Unit.id')) + parameter_id = Column(Integer(), ForeignKey('parameter.id')) + alternate_unit_id = Column(Integer(), ForeignKey('unit.id')) def __repr__(self): @@ -719,7 +721,7 @@ class ResourceMorphology(Base): __tablename__ = 'resource_morphology' id = Column(Integer(), primary_key=True, nullable=False ) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) morphology_uri = Column(Text()) @@ -739,7 +741,7 @@ class DataSource(BaseEntity): name = Column(Text()) description = Column(Text()) - data_source_type_id = Column(Integer(), ForeignKey('DataSourceType.id')) + data_source_type_id = Column(Integer(), ForeignKey('data_source_type.id')) full_title = Column(Text()) creator = Column(Text()) subject = Column(Text()) @@ -752,7 +754,7 @@ class DataSource(BaseEntity): language = Column(Text()) relation = Column(Text()) temporal_coverage = Column(Text()) - location_coverage_id = Column(Integer(), ForeignKey('LocationResolution.id')) + location_coverage_id = Column(Integer(), ForeignKey('location_resolution.id')) rights = Column(Text()) license = Column(Text()) uri = Column(Text()) @@ -783,7 +785,7 @@ class DataSourceType(BaseEntity): """ __tablename__ = 'data_source_type' - source_type_id = Column(Integer(), ForeignKey('SourceType.id')) + source_type_id = Column(Integer(), ForeignKey('source_type.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -810,7 +812,7 @@ class FileObjectMetadata(BaseEntity): """ __tablename__ = 'file_object_metadata' - data_source_id = Column(Integer(), ForeignKey('DataSource.id')) + data_source_id = Column(Integer(), ForeignKey('data_source.id')) bucket_path = Column(Text()) file_format = Column(Text()) file_size = Column(Integer()) @@ -892,7 +894,7 @@ class Equipment(LookupBase): """ __tablename__ = 'equipment' - equipment_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + equipment_location_id = Column(Integer(), ForeignKey('location_address.id')) id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) description = Column(Text()) @@ -918,11 +920,11 @@ class Experiment(BaseEntity): """ __tablename__ = 'experiment' - analyst_id = Column(Integer(), ForeignKey('Contact.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) exper_start_date = Column(Date()) exper_duration = Column(Numeric()) - exper_duration_unit_id = Column(Integer(), ForeignKey('Unit.id')) - exper_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + exper_duration_unit_id = Column(Integer(), ForeignKey('unit.id')) + exper_location_id = Column(Integer(), ForeignKey('location_address.id')) description = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -976,12 +978,12 @@ class Method(BaseEntity): __tablename__ = 'method' name = Column(Text()) - method_abbrev_id = Column(Integer(), ForeignKey('MethodAbbrev.id')) - method_category_id = Column(Integer(), ForeignKey('MethodCategory.id')) - method_standard_id = Column(Integer(), ForeignKey('MethodStandard.id')) + method_abbrev_id = Column(Integer(), ForeignKey('method_abbrev.id')) + method_category_id = Column(Integer(), ForeignKey('method_category.id')) + method_standard_id = Column(Integer(), ForeignKey('method_standard.id')) description = Column(Text()) detection_limits = Column(Text()) - source_id = Column(Integer(), ForeignKey('DataSource.id')) + source_id = Column(Integer(), ForeignKey('data_source.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1009,10 +1011,10 @@ class PreparedSample(BaseEntity): __tablename__ = 'prepared_sample' name = Column(Text()) - field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) - prep_method_id = Column(Integer(), ForeignKey('PreparationMethod.id')) + field_sample_id = Column(Integer(), ForeignKey('field_sample.id')) + prep_method_id = Column(Integer(), ForeignKey('preparation_method.id')) prep_date = Column(Date()) - preparer_id = Column(Integer(), ForeignKey('Contact.id')) + preparer_id = Column(Integer(), ForeignKey('contact.id')) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -1041,9 +1043,9 @@ class Resource(BaseEntity): __tablename__ = 'resource' name = Column(Text()) - primary_ag_product_id = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) - resource_class_id = Column(Integer(), ForeignKey('ResourceClass.id')) - resource_subclass_id = Column(Integer(), ForeignKey('ResourceSubclass.id')) + primary_ag_product_id = Column(Integer(), ForeignKey('primary_ag_product.id')) + resource_class_id = Column(Integer(), ForeignKey('resource_class.id')) + resource_subclass_id = Column(Integer(), ForeignKey('resource_subclass.id')) note = Column(Text()) test = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1122,18 +1124,18 @@ class BillionTon2023Record(BaseEntity): """ __tablename__ = 'billion_ton2023_record' - subclass_id = Column(Integer(), ForeignKey('ResourceSubclass.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + subclass_id = Column(Integer(), ForeignKey('resource_subclass.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) geoid = Column(Text()) county_square_miles = Column(Float()) model_name = Column(Text()) scenario_name = Column(Text()) price_offered_usd = Column(Numeric()) production = Column(Integer()) - production_unit_id = Column(Integer(), ForeignKey('Unit.id')) + production_unit_id = Column(Integer(), ForeignKey('unit.id')) btu_ton = Column(Integer()) production_energy_content = Column(Integer()) - energy_content_unit_id = Column(Integer(), ForeignKey('Unit.id')) + energy_content_unit_id = Column(Integer(), ForeignKey('unit.id')) product_density_dtpersqmi = Column(Numeric()) land_source = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1187,12 +1189,12 @@ class LandiqRecord(BaseEntity): """ __tablename__ = 'landiq_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - polygon_id = Column(Integer(), ForeignKey('Polygon.id')) - main_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) - secondary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) - tertiary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) - quaternary_crop = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + polygon_id = Column(Integer(), ForeignKey('polygon.id')) + main_crop = Column(Integer(), ForeignKey('primary_ag_product.id')) + secondary_crop = Column(Integer(), ForeignKey('primary_ag_product.id')) + tertiary_crop = Column(Integer(), ForeignKey('primary_ag_product.id')) + quaternary_crop = Column(Integer(), ForeignKey('primary_ag_product.id')) confidence = Column(Integer()) irrigated = Column(Boolean()) acres = Column(Float()) @@ -1226,7 +1228,7 @@ class UsdaCommodity(LookupBase): usda_source = Column(Text()) usda_code = Column(Text()) - parent_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + parent_commodity_id = Column(Integer(), ForeignKey('usda_commodity.id')) id = Column(Integer(), primary_key=True, nullable=False ) name = Column(Text()) description = Column(Text()) @@ -1252,9 +1254,9 @@ class ResourceUsdaCommodityMap(BaseEntity): """ __tablename__ = 'resource_usda_commodity_map' - resource_id = Column(Integer(), ForeignKey('Resource.id')) - primary_ag_product_id = Column(Integer(), ForeignKey('PrimaryAgProduct.id')) - usda_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) + primary_ag_product_id = Column(Integer(), ForeignKey('primary_ag_product.id')) + usda_commodity_id = Column(Integer(), ForeignKey('usda_commodity.id')) match_tier = Column(Text()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1283,9 +1285,9 @@ class UsdaCensusRecord(BaseEntity): """ __tablename__ = 'usda_census_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) geoid = Column(Text()) - commodity_code = Column(Integer(), ForeignKey('UsdaCommodity.id')) + commodity_code = Column(Integer(), ForeignKey('usda_commodity.id')) year = Column(Integer()) source_reference = Column(Text()) note = Column(Text()) @@ -1345,9 +1347,9 @@ class UsdaMarketReport(BaseEntity): report_series_title = Column(Text()) frequency = Column(Text()) office_name = Column(Text()) - office_city_id = Column(Integer(), ForeignKey('LocationAddress.id')) + office_city_id = Column(Integer(), ForeignKey('location_address.id')) office_state_fips = Column(Text()) - source_id = Column(Integer(), ForeignKey('DataSource.id')) + source_id = Column(Integer(), ForeignKey('data_source.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1374,12 +1376,12 @@ class UsdaMarketRecord(BaseEntity): """ __tablename__ = 'usda_market_record' - report_id = Column(Integer(), ForeignKey('UsdaMarketReport.id')) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + report_id = Column(Integer(), ForeignKey('usda_market_report.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) report_begin_date = Column(DateTime()) report_end_date = Column(DateTime()) report_date = Column(DateTime()) - commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + commodity_id = Column(Integer(), ForeignKey('usda_commodity.id')) market_type_id = Column(Integer()) market_type_category = Column(Text()) grp = Column(Text()) @@ -1391,7 +1393,7 @@ class UsdaMarketRecord(BaseEntity): application = Column(Text()) pkg = Column(Text()) sale_type = Column(Text()) - price_unit_id = Column(Integer(), ForeignKey('Unit.id')) + price_unit_id = Column(Integer(), ForeignKey('unit.id')) freight = Column(Text()) trans_mode = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1420,7 +1422,7 @@ class LocationAddress(BaseEntity): """ __tablename__ = 'location_address' - geography_id = Column(Text(), ForeignKey('Geography.geoid')) + geography_id = Column(Text(), ForeignKey('geography.geoid')) address_line1 = Column(Text()) address_line2 = Column(Text()) city = Column(Text()) @@ -1504,11 +1506,11 @@ class UsdaSurveyRecord(BaseEntity): """ __tablename__ = 'usda_survey_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) geoid = Column(Text()) - commodity_code = Column(Integer(), ForeignKey('UsdaCommodity.id')) + commodity_code = Column(Integer(), ForeignKey('usda_commodity.id')) year = Column(Integer()) - survey_program_id = Column(Integer(), ForeignKey('UsdaSurveyProgram.id')) + survey_program_id = Column(Integer(), ForeignKey('usda_survey_program.id')) survey_period = Column(Text()) reference_month = Column(Text()) seasonal_flag = Column(Boolean()) @@ -1542,7 +1544,7 @@ class UsdaTermMap(BaseEntity): source_system = Column(Text()) source_context = Column(Text()) raw_term = Column(Text()) - usda_commodity_id = Column(Integer(), ForeignKey('UsdaCommodity.id')) + usda_commodity_id = Column(Integer(), ForeignKey('usda_commodity.id')) is_verified = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -1622,22 +1624,22 @@ class FieldSample(BaseEntity): __tablename__ = 'field_sample' name = Column(Text()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) - provider_id = Column(Integer(), ForeignKey('Provider.id')) - collector_id = Column(Integer(), ForeignKey('Contact.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) + provider_id = Column(Integer(), ForeignKey('provider.id')) + collector_id = Column(Integer(), ForeignKey('contact.id')) sample_collection_source = Column(Text()) amount_collected = Column(Numeric()) - amount_collected_unit_id = Column(Integer(), ForeignKey('Unit.id')) - sampling_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) - field_storage_method_id = Column(Integer(), ForeignKey('FieldStorageMethod.id')) + amount_collected_unit_id = Column(Integer(), ForeignKey('unit.id')) + sampling_location_id = Column(Integer(), ForeignKey('location_address.id')) + field_storage_method_id = Column(Integer(), ForeignKey('field_storage_method.id')) field_storage_duration_value = Column(Numeric()) - field_storage_duration_unit_id = Column(Integer(), ForeignKey('Unit.id')) - field_storage_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + field_storage_duration_unit_id = Column(Integer(), ForeignKey('unit.id')) + field_storage_location_id = Column(Integer(), ForeignKey('location_address.id')) collection_timestamp = Column(DateTime()) - collection_method_id = Column(Integer(), ForeignKey('CollectionMethod.id')) - harvest_method_id = Column(Integer(), ForeignKey('HarvestMethod.id')) + collection_method_id = Column(Integer(), ForeignKey('collection_method.id')) + harvest_method_id = Column(Integer(), ForeignKey('harvest_method.id')) harvest_date = Column(Date()) - field_sample_storage_location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + field_sample_storage_location_id = Column(Integer(), ForeignKey('location_address.id')) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -1690,11 +1692,11 @@ class FieldSampleCondition(BaseEntity): """ __tablename__ = 'field_sample_condition' - field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) - ag_treatment_id = Column(Integer(), ForeignKey('AgTreatment.id')) + field_sample_id = Column(Integer(), ForeignKey('field_sample.id')) + ag_treatment_id = Column(Integer(), ForeignKey('ag_treatment.id')) last_application_date = Column(Date()) treatment_amount_per_acre = Column(Float()) - processing_method_id = Column(Integer(), ForeignKey('ProcessingMethod.id')) + processing_method_id = Column(Integer(), ForeignKey('processing_method.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1796,8 +1798,8 @@ class LocationSoilType(BaseEntity): """ __tablename__ = 'location_soil_type' - location_id = Column(Integer(), ForeignKey('LocationAddress.id')) - soil_type_id = Column(Integer(), ForeignKey('SoilType.id')) + location_id = Column(Integer(), ForeignKey('location_address.id')) + soil_type_id = Column(Integer(), ForeignKey('soil_type.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1824,11 +1826,11 @@ class PhysicalCharacteristic(BaseEntity): """ __tablename__ = 'physical_characteristic' - field_sample_id = Column(Integer(), ForeignKey('FieldSample.id')) + field_sample_id = Column(Integer(), ForeignKey('field_sample.id')) particle_length = Column(Numeric()) particle_width = Column(Numeric()) particle_height = Column(Numeric()) - particle_unit_id = Column(Integer(), ForeignKey('Unit.id')) + particle_unit_id = Column(Integer(), ForeignKey('unit.id')) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) updated_at = Column(DateTime()) @@ -1857,7 +1859,7 @@ class Dataset(BaseEntity): name = Column(Text()) record_type = Column(Text()) - source_id = Column(Integer(), ForeignKey('DataSource.id')) + source_id = Column(Integer(), ForeignKey('data_source.id')) start_date = Column(Date()) end_date = Column(Date()) description = Column(Text()) @@ -1912,15 +1914,15 @@ class Observation(BaseEntity): """ __tablename__ = 'observation' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) record_type = Column(Text()) record_id = Column(Integer()) - parameter_id = Column(Integer(), ForeignKey('Parameter.id')) + parameter_id = Column(Integer(), ForeignKey('parameter.id')) value = Column(Numeric()) - unit_id = Column(Integer(), ForeignKey('Unit.id')) - dimension_type_id = Column(Integer(), ForeignKey('DimensionType.id')) + unit_id = Column(Integer(), ForeignKey('unit.id')) + dimension_type_id = Column(Integer(), ForeignKey('dimension_type.id')) dimension_value = Column(Numeric()) - dimension_unit_id = Column(Integer(), ForeignKey('Unit.id')) + dimension_unit_id = Column(Integer(), ForeignKey('unit.id')) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) created_at = Column(DateTime()) @@ -1948,11 +1950,11 @@ class FacilityRecord(BaseEntity): """ __tablename__ = 'facility_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) facility_name = Column(Text()) - location_id = Column(Integer(), ForeignKey('LocationAddress.id')) + location_id = Column(Integer(), ForeignKey('location_address.id')) capacity_mw = Column(Numeric()) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) operator = Column(Text()) start_year = Column(Integer()) note = Column(Text()) @@ -2058,7 +2060,7 @@ class Parameter(BaseEntity): __tablename__ = 'parameter' name = Column(Text()) - standard_unit_id = Column(Integer(), ForeignKey('Unit.id')) + standard_unit_id = Column(Integer(), ForeignKey('unit.id')) calculated = Column(Boolean()) description = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2169,15 +2171,15 @@ class Aim1RecordBase(BaseEntity): """ __tablename__ = 'aim1_record_base' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2206,15 +2208,15 @@ class Aim2RecordBase(BaseEntity): """ __tablename__ = 'aim2_record_base' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2269,7 +2271,7 @@ class ResourceAvailability(BaseEntity): """ __tablename__ = 'resource_availability' - resource_id = Column(Integer(), ForeignKey('Resource.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) geoid = Column(Text()) from_month = Column(Integer()) to_month = Column(Integer()) @@ -2328,7 +2330,7 @@ class PreparationMethod(BaseEntity): name = Column(Text()) description = Column(Text()) - prep_method_abbrev_id = Column(Integer(), ForeignKey('PreparationMethodAbbreviation.id')) + prep_method_abbrev_id = Column(Integer(), ForeignKey('preparation_method_abbreviation.id')) prep_temp_c = Column(Numeric()) uri = Column(Text()) drying_step = Column(Boolean()) @@ -2383,15 +2385,15 @@ class ProximateRecord(Aim1RecordBase): """ __tablename__ = 'proximate_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2420,15 +2422,15 @@ class UltimateRecord(Aim1RecordBase): """ __tablename__ = 'ultimate_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2457,15 +2459,15 @@ class CompositionalRecord(Aim1RecordBase): """ __tablename__ = 'compositional_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2494,15 +2496,15 @@ class IcpRecord(Aim1RecordBase): """ __tablename__ = 'icp_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2535,15 +2537,15 @@ class XrfRecord(Aim1RecordBase): intensity = Column(Numeric()) energy_slope = Column(Numeric()) energy_offset = Column(Numeric()) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2574,15 +2576,15 @@ class XrdRecord(Aim1RecordBase): scan_low_nm = Column(Integer()) scan_high_nm = Column(Integer()) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2611,15 +2613,15 @@ class CalorimetryRecord(Aim1RecordBase): """ __tablename__ = 'calorimetry_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2648,15 +2650,15 @@ class FtnirRecord(Aim1RecordBase): """ __tablename__ = 'ftnir_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2685,15 +2687,15 @@ class RgbRecord(Aim1RecordBase): """ __tablename__ = 'rgb_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_no = Column(Integer()) technical_replicate_total = Column(Integer()) - method_id = Column(Integer(), ForeignKey('Method.id')) - analyst_id = Column(Integer(), ForeignKey('Contact.id')) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + method_id = Column(Integer(), ForeignKey('method.id')) + analyst_id = Column(Integer(), ForeignKey('contact.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2728,15 +2730,15 @@ class PretreatmentRecord(Aim2RecordBase): block_position = Column(Text()) temperature = Column(Numeric()) replicate_no = Column(Integer()) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2773,15 +2775,15 @@ class FermentationRecord(Aim2RecordBase): agitation_rpm = Column(Numeric()) vessel_id = Column(Integer()) analyte_detection_equipment_id = Column(Integer()) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2813,15 +2815,15 @@ class GasificationRecord(Aim2RecordBase): feedstock_mass = Column(Numeric()) bed_temperature = Column(Numeric()) gas_flow_rate = Column(Numeric()) - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) @@ -2850,15 +2852,15 @@ class AutoclaveRecord(Aim2RecordBase): """ __tablename__ = 'autoclave_record' - dataset_id = Column(Integer(), ForeignKey('DataSource.id')) - experiment_id = Column(Integer(), ForeignKey('Experiment.id')) - resource_id = Column(Integer(), ForeignKey('Resource.id')) + dataset_id = Column(Integer(), ForeignKey('data_source.id')) + experiment_id = Column(Integer(), ForeignKey('experiment.id')) + resource_id = Column(Integer(), ForeignKey('resource.id')) sample_id = Column(Integer()) technical_replicate_total = Column(Integer()) technical_replicate_no = Column(Integer()) calc_method_id = Column(Integer()) calc_analyst_id = Column(Integer()) - raw_data_id = Column(Integer(), ForeignKey('FileObjectMetadata.id')) + raw_data_id = Column(Integer(), ForeignKey('file_object_metadata.id')) qc_pass = Column(Boolean()) note = Column(Text()) id = Column(Integer(), primary_key=True, nullable=False ) diff --git a/src/ca_biositing/datamodels/utils/generate_sqla.py b/src/ca_biositing/datamodels/utils/generate_sqla.py index 929d5298..ef48e08a 100644 --- a/src/ca_biositing/datamodels/utils/generate_sqla.py +++ b/src/ca_biositing/datamodels/utils/generate_sqla.py @@ -26,9 +26,40 @@ def replace_tablename(match): new_content = re.sub(pattern, replace_tablename, content) + # Convert ForeignKey references from class names to table names + # Matches: ForeignKey('SomeClass.id') -> ForeignKey('some_class.id') + # Convert ForeignKey references from class names to table names + # Matches: ForeignKey('SomeClass.column') -> ForeignKey('some_class.column') + def replace_foreign_key(match): + fk_content = match.group(0) + # Extract the table/class name and column name from inside the quotes + inner_match = re.search(r"ForeignKey\('([A-Za-z0-9]+)\.([A-Za-z0-9]+)'\)", fk_content) + if inner_match: + class_name = inner_match.group(1) + column_name = inner_match.group(2) + table_name = to_snake_case(class_name) + return fk_content.replace(f"'{class_name}.{column_name}'", f"'{table_name}.{column_name}'") + return fk_content + + # Apply to all ForeignKey occurrences + new_content = re.sub(r"ForeignKey\('[^']+\.[^']+'\)", replace_foreign_key, new_content) + + # Inject the Base import + new_content = "from ...database import Base\n\n" + new_content + + # Inject the Base import + new_content = "from ...database import Base\n\n" + new_content + + # Remove the line that redefines Base + new_content = re.sub(r"Base = declarative_base\(\)\n", "", new_content) + + # Remove the unused import if present + new_content = re.sub(r"from sqlalchemy\.orm import declarative_base\n", "", new_content) + with open(file_path, 'w') as f: f.write(new_content) + def generate_sqla(): """ Generates SQLAlchemy models from LinkML schema modules. From b08f15ec6d86be05c868d576b712dca99cf740f8 Mon Sep 17 00:00:00 2001 From: Avi <32148378+avi9664@users.noreply.github.com> Date: Sat, 10 Jan 2026 20:35:47 -0800 Subject: [PATCH 10/10] modified engine.py to get correct database url from env --- alembic/env.py | 2 +- docs/README.md | 2 +- docs/datamodels/README.md | 2 +- docs/deployment/README.md | 2 +- docs/pipeline/README.md | 2 +- docs/resources/README.md | 2 +- docs/webservice/README.md | 2 +- .../.ipynb_checkpoints/engine-checkpoint.py | 32 + .../etl_notebook-checkpoint.ipynb | 318 ++++ ...sheet_extraction_notebook-checkpoint.ipynb | 899 ++++++++++ .../ca_biositing/pipeline/utils/engine.py | 28 +- .../pipeline/utils/etl_notebook.ipynb | 1497 +---------------- .../utils/gsheet_extraction_notebook.ipynb | 1083 +++--------- 13 files changed, 1509 insertions(+), 2362 deletions(-) create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/engine-checkpoint.py create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/etl_notebook-checkpoint.ipynb create mode 100644 src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/gsheet_extraction_notebook-checkpoint.ipynb diff --git a/alembic/env.py b/alembic/env.py index ad465742..8df293e1 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -26,7 +26,7 @@ # from ca_biositing.datamodels.user import * from ca_biositing.datamodels.schemas.generated import ca_biositing from ca_biositing.datamodels.schemas.generated.ca_biositing import * -from ca_biositing.datamodels.database import Base +# from ca_biositing.datamodels.database import Base # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/docs/README.md b/docs/README.md index 32d46ee8..94389aee 120000 --- a/docs/README.md +++ b/docs/README.md @@ -1 +1 @@ -../README.md \ No newline at end of file +../README.md diff --git a/docs/datamodels/README.md b/docs/datamodels/README.md index b62101ad..f3c7aa23 120000 --- a/docs/datamodels/README.md +++ b/docs/datamodels/README.md @@ -1 +1 @@ -../../src/ca_biositing/datamodels/README.md \ No newline at end of file +../../src/ca_biositing/datamodels/README.md diff --git a/docs/deployment/README.md b/docs/deployment/README.md index d5c46ad4..5159bcfe 120000 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -1 +1 @@ -../../deployment/README.md \ No newline at end of file +../../deployment/README.md diff --git a/docs/pipeline/README.md b/docs/pipeline/README.md index 82a8c831..544d50b1 120000 --- a/docs/pipeline/README.md +++ b/docs/pipeline/README.md @@ -1 +1 @@ -../../src/ca_biositing/pipeline/README.md \ No newline at end of file +../../src/ca_biositing/pipeline/README.md diff --git a/docs/resources/README.md b/docs/resources/README.md index 9f21ec35..ea34ca59 120000 --- a/docs/resources/README.md +++ b/docs/resources/README.md @@ -1 +1 @@ -../../resources/README.md \ No newline at end of file +../../resources/README.md diff --git a/docs/webservice/README.md b/docs/webservice/README.md index 749a635e..24ea2883 120000 --- a/docs/webservice/README.md +++ b/docs/webservice/README.md @@ -1 +1 @@ -../../src/ca_biositing/webservice/README.md \ No newline at end of file +../../src/ca_biositing/webservice/README.md diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/engine-checkpoint.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/engine-checkpoint.py new file mode 100644 index 00000000..c5349511 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/engine-checkpoint.py @@ -0,0 +1,32 @@ +from sqlmodel import create_engine, Session +import os +from dotenv import load_dotenv + +#This module queries the db via the ORM + +# Get the root +path = os.getcwd() +project_root = None +while path != os.path.dirname(path): # Stop at the filesystem root + if 'pixi.toml' in os.listdir(path): + project_root = path + break + path = os.path.dirname(path) + +load_dotenv(dotenv_path=project_root + "\\resources\\docker\\.env") + +# Database Connection +POSTGRES_USER = os.getenv("POSTGRES_USER") +POSTGRES_PASSWORD= os.getenv("POSTGRES_PASSWORD") + +# 2. Host Port Mapping +# This is the port on your local machine that will connect to the container's port 5432. +POSTGRES_PORT= os.getenv("POSTGRES_PORT") + +DATABASE_URL = f"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/biocirv_db" + +# old: +# DATABASE_URL = "postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db" +engine = create_engine(DATABASE_URL) + +db_session = Session(engine) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/etl_notebook-checkpoint.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/etl_notebook-checkpoint.ipynb new file mode 100644 index 00000000..3c8228d6 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/etl_notebook-checkpoint.ipynb @@ -0,0 +1,318 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ETL Notebook for CA Biositing Project\n", + "\n", + "This notebook provides a documented walkthrough of the ETL (Extract, Transform, Load) process for the CA Biositing project. It is designed for interactive development and exploration before migrating logic into the production pipeline.\n", + "\n", + "It covers:\n", + "\n", + "1. **Setup**: Importing necessary libraries and establishing a connection to the database.\n", + "2. **Extraction**: Pulling raw data from Google Sheets.\n", + "3. **Cleaning**: Standardizing data types, handling missing values, and cleaning column names.\n", + "4. **Normalization**: Replacing human-readable names (e.g., \"Corn\") with database foreign key IDs (e.g., `resource_id: 1`).\n", + "5. **Utilities**: Common functions for data manipulation and analysis.\n", + "6. **Deployment Plan**: A step-by-step guide for moving the code from this notebook into the production ETL modules." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "import logging\n", + "from IPython.display import display\n", + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "\n", + "# --- Basic Logging Configuration for Notebook ---\n", + "# When running in a notebook, we use Python's standard logging.\n", + "# In the production pipeline, this will be replaced by Prefect's `get_run_logger()`\n", + "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n", + "logger = logging.getLogger()\n", + "\n", + "# --- Robustly find the project root ---\n", + "# This ensures that the notebook can be run from any directory within the project.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# Add the project root to the Python path to allow for module imports\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " logger.info(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " logger.info(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import project modules ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + " from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import replace_name_with_id_df\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " logger.info('Successfully imported all project modules.')\n", + "except ImportError as e:\n", + " logger.error(f'Failed to import project modules: {e}', exc_info=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Cleaning Function" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " \"\"\"Cleans and standardizes a DataFrame extracted from Google Sheets.\n", + "\n", + " This function performs several key operations:\n", + " 1. Cleans column names to a standard format (snake_case).\n", + " 2. Drops rows where essential columns ('repl_no', 'value') are empty.\n", + " 3. Coerces data types for numeric and datetime columns, handling errors gracefully.\n", + " 4. Converts remaining columns to the best possible data types.\n", + "\n", + " Args:\n", + " df (pd.DataFrame): The raw DataFrame.\n", + "\n", + " Returns:\n", + " pd.DataFrame: The cleaned DataFrame.\n", + " \"\"\"\n", + " logger.info('Starting DataFrame cleaning process.')\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.error('Input is not a pandas DataFrame.')\n", + " return None\n", + " \n", + " try:\n", + " # 1. Clean names and drop rows with missing essential data\n", + " df_cleaned = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + " logger.info(f'Dropped {len(df) - len(df_cleaned)} rows with missing values.')\n", + "\n", + " # 2. Coerce numeric types\n", + " df_cleaned['repl_no'] = pd.to_numeric(df_cleaned['repl_no'], errors='coerce').astype('Int32')\n", + " df_cleaned['value'] = pd.to_numeric(df_cleaned['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Coerce datetime types\n", + " if 'created_at' in df_cleaned.columns:\n", + " df_cleaned['created_at'] = pd.to_datetime(df_cleaned['created_at'], errors='coerce')\n", + " if 'updated_at' in df_cleaned.columns:\n", + " df_cleaned['updated_at'] = pd.to_datetime(df_cleaned['updated_at'], errors='coerce')\n", + "\n", + " # 4. Replace empty strings with NaN so they are properly ignored\n", + " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", + "\n", + " # 5. Convert other dtypes to best possible\n", + " df_cleaned = df_cleaned.convert_dtypes()\n", + " logger.info('Successfully cleaned DataFrame.')\n", + "\n", + " # 6. Convert all string data to lowercase\n", + " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + " logger.info('Converted all string data to lowercase.')\n", + " return df_cleaned\n", + "\n", + " except Exception as e:\n", + " logger.error(f'An error occurred during DataFrame cleaning: {e}', exc_info=True)\n", + " return None\n", + "\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Normalization Function" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_dataframes(dataframes, normalize_columns):\n", + " \"\"\"Normalizes a list of DataFrames by replacing name columns with foreign key IDs.\n", + "\n", + " This function iterates through a list of dataframes and, for each one, iterates\n", + " through a dictionary of columns that need to be normalized. It uses the \n", + " `replace_name_with_id_df` utility to look up or create the corresponding ID\n", + " in the database.\n", + "\n", + " Args:\n", + " dataframes (list[pd.DataFrame]): A list of DataFrames to normalize.\n", + " normalize_columns (dict): A dictionary mapping column names to SQLModel classes and attributes.\n", + "\n", + " Returns:\n", + " list[pd.DataFrame]: The list of normalized DataFrames.\n", + " \"\"\"\n", + " logger.info(f'Starting normalization process for {len(dataframes)} dataframes.')\n", + " normalized_dfs = []\n", + " try:\n", + " with Session(engine) as db:\n", + " for i, df in enumerate(dataframes):\n", + " if not isinstance(df, pd.DataFrame):\n", + " logger.warning(f'Item {i+1} is not a DataFrame, skipping.')\n", + " continue\n", + " \n", + " logger.info(f'Processing DataFrame #{i+1} with {len(df)} rows.')\n", + " df_normalized = df.copy()\n", + "\n", + " for df_col, (model, model_name_attr) in normalize_columns.items():\n", + " if df_col not in df_normalized.columns:\n", + " logger.warning(f\"Column '{df_col}' not in DataFrame #{i+1}. Skipping normalization for this column.\")\n", + " continue\n", + " \n", + " try:\n", + " # Skip normalization if the column is all NaN/None\n", + " if df_normalized[df_col].isnull().all():\n", + " logger.info(f\"Skipping normalization for column '{df_col}' as it contains only null values.\")\n", + " continue\n", + " \n", + " logger.info(f\"Normalizing column '{df_col}' using model '{model.__name__}'.\")\n", + " df_normalized, num_created = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name='id',\n", + " final_column_name=f'{df_col}_id'\n", + " )\n", + " if num_created > 0:\n", + " logger.info(f\"Created {num_created} new records in '{model.__name__}' table.\")\n", + " new_col_name = f'{df_col}_id'\n", + " num_nulls = df_normalized[new_col_name].isnull().sum()\n", + " logger.info(f\"Successfully normalized '{df_col}'. New column '{new_col_name}' contains {num_nulls} null values.\")\n", + " except Exception as e:\n", + " logger.error(f\"Error normalizing column '{df_col}' in DataFrame #{i+1}: {e}\", exc_info=True)\n", + " continue # Continue to the next column\n", + " \n", + " normalized_dfs.append(df_normalized)\n", + " logger.info(f'Finished processing DataFrame #{i+1}.')\n", + " \n", + " logger.info('Committing database session.')\n", + " db.commit()\n", + " logger.info('Database commit successful.')\n", + " except Exception as e:\n", + " logger.error(f'A critical error occurred during the database session: {e}', exc_info=True)\n", + " db.rollback()\n", + " logger.info('Database session rolled back.')\n", + " \n", + " return normalized_dfs\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ETL Execution Example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# --- 1. Extraction ---\n", + "# In a real Prefect flow, each extraction would be a separate task.\n", + "logger.info('Starting data extraction...')\n", + "prox_df = proximate.extract(project_root=project_root)\n", + "ult_df = ultimate.extract(project_root=project_root)\n", + "cmp_df = cmpana.extract(project_root=project_root)\n", + "dataframes = [prox_df, ult_df, cmp_df]\n", + "logger.info('Data extraction complete.')\n", + "\n", + "# --- 2. Cleaning ---\n", + "# This list comprehension applies the cleaning function to each extracted dataframe.\n", + "logger.info('Starting data cleaning...')\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes if df is not None]\n", + "logger.info('Data cleaning complete.')\n", + "\n", + "# --- 3. Normalization ---\n", + "# This dictionary defines the columns to be normalized. \n", + "# The key is the column name in the DataFrame.\n", + "# The value is a tuple containing the corresponding SQLAlchemy model and the name of the attribute on the model to match against.\n", + "NORMALIZE_COLUMNS = {\n", + " 'resource': (Resource, 'name'),\n", + " 'prepared_sample': (PreparedSample, 'name'),\n", + " 'preparation_method': (PreparationMethod, 'name'),\n", + " 'parameter': (Parameter, 'name'),\n", + " 'unit': (Unit, 'name'),\n", + " 'analyst_email': (Contact, 'email'),\n", + " 'analysis_type': (AnalysisType, 'name'),\n", + " 'primary_ag_product': (PrimaryAgProduct, 'name')\n", + "}\n", + "\n", + "logger.info('Starting data normalization...')\n", + "normalized_dataframes = normalize_dataframes(clean_dataframes, NORMALIZE_COLUMNS)\n", + "logger.info('Data normalization complete.')\n", + "\n", + "# --- 4. Display Results ---\n", + "logger.info('Displaying results of normalization...')\n", + "for i, df in enumerate(normalized_dataframes):\n", + " print(f'--- Normalized DataFrame {i+1} ---')\n", + " display(df.head())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deployment Plan\n", + "\n", + "The code in this notebook will be transitioned to the main ETL pipeline by following these steps:\n", + "\n", + "1. **Function Migration**: The `clean_the_gsheets` and `normalize_dataframes` functions will be moved to a new utility module, for example, `src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_utils.py`. Each function will be decorated with `@task` from Prefect to turn it into a reusable pipeline component.\n", + "2. **Flow Creation**: A new Prefect flow will be created in the `src/ca_biositing/pipeline/ca_biositing/pipeline/flows/` directory (e.g., `master_extraction_flow.py`). This flow will orchestrate the entire ETL process for a given data source.\n", + "3. **Task Integration**: The new flow will be composed of individual tasks. It will call the existing extraction tasks (`proximate.extract`, etc.), and then pass the results to the new cleaning and normalization tasks from `etl_utils.py`.\n", + "4. **Logging**: The `logging` module will be replaced with `get_run_logger()` from Prefect within the tasks to ensure logs are captured by the Prefect UI.\n", + "5. **Configuration**: The `NORMALIZE_COLUMNS` dictionary will be moved to a configuration file or defined within the relevant flow to make it easier to manage and modify without changing the code.\n", + "6. **Testing**: Unit tests will be written for the new utility functions in `etl_utils.py`. An integration test will be created for the new Prefect flow to ensure all the tasks work together correctly.\n", + "7. **Deployment**: Once the flow is complete and tested, it will be deployed to the Prefect server using the `pixi run deploy` command, making it available to be run on a schedule or manually via the UI." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/gsheet_extraction_notebook-checkpoint.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/gsheet_extraction_notebook-checkpoint.ipynb new file mode 100644 index 00000000..b9feba10 --- /dev/null +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/.ipynb_checkpoints/gsheet_extraction_notebook-checkpoint.ipynb @@ -0,0 +1,899 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import pandas as pd\n", + "import numpy as np\n", + "import janitor as jn\n", + "from IPython.display import display\n", + "\n", + "# --- Robustly find the project root ---\n", + "# The project root is the directory containing the 'pixi.toml' file.\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path): # Stop at the filesystem root\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "# --- Add project root to sys.path ---\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + " print(f\"Added project root '{project_root}' to sys.path\")\n", + "else:\n", + " print(f\"Project root '{project_root}' is already in sys.path\")\n", + "\n", + "# --- Import the module ---\n", + "try:\n", + " from src.ca_biositing.pipeline.ca_biositing.pipeline.etl.extract import proximate, ultimate, cmpana\n", + " print(\"Successfully imported all module.\")\n", + "except ImportError as e:\n", + " print(f\"Failed to import modules: {e}\")\n", + " print(f\"\\nFull sys.path: {sys.path}\")\n", + "\n", + "# --- Run the extraction ---\n", + "if 'proximate' in locals():\n", + " try:\n", + " # Pass the project_root to the extract function\n", + " df = proximate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted proximate data.\")\n", + " display(df.head())\n", + " else:\n", + " print(\"\\n Prox extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during prox extraction: {e}\")\n", + "\n", + "if 'ultimate' in locals():\n", + " try:\n", + " df2 = ultimate.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted Ultimate data.\")\n", + " display(df2.head())\n", + " else:\n", + " print(\"\\n Ultimate extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during extraction: {e}\")\n", + "\n", + "if 'cmpana' in locals():\n", + " try:\n", + " df3 = cmpana.extract(project_root=project_root)\n", + " if df is not None:\n", + " print(\"\\nSuccessfully extracted CmpAna data.\")\n", + " display(df3.head())\n", + " else:\n", + " print(\"\\nCmpAna extraction returned no data. Check the logs above for errors.\")\n", + " except Exception as e:\n", + " print(f\"\\nAn error occurred during cmp ana extraction: {e}\")\n", + " finally:\n", + " print(\"\\nCmp Ana extraction process completed.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This function seeks to clean the incoming gsheet dataframes and coerce the types" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_the_gsheets(df):\n", + " # 1. Clean names and drop rows\n", + " df = df.clean_names().dropna(subset=['repl_no', 'value'])\n", + "\n", + " # 2. Coerce types (using errors='coerce' handles messy string data)\n", + " df['repl_no'] = pd.to_numeric(df['repl_no'], errors='coerce').astype('Int32') # Capital 'I' handles NaNs\n", + " df['value'] = pd.to_numeric(df['value'], errors='coerce').astype(np.float32)\n", + "\n", + " # 3. Dates\n", + " df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')\n", + " df['updated_at'] = pd.to_datetime(df['updated_at'], errors='coerce')\n", + "\n", + " # 4. Convert remaining objects to best possible types (like strings)\n", + " df = df.convert_dtypes()\n", + "\n", + " # 5. Convert all string data to lowercase\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "\n", + " # 6. Convert empty strings to NaN\n", + " df.replace(\"\", np.nan, inplace=True)\n", + "\n", + " return df # Return the FULL dataframe, not just .head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlmodel import Session, select, create_engine\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "from dotenv import load_dotenv\n", + "\n", + "#This module queries the db via the ORM\n", + "\n", + "load_dotenv(dotenv_path=project_root + \"\\\\resources\\\\docker\\\\.env\")\n", + "\n", + "# Database Connection\n", + "\n", + "POSTGRES_USER = os.getenv(\"POSTGRES_USER\")\n", + "POSTGRES_PASSWORD= os.getenv(\"POSTGRES_PASSWORD\")\n", + "\n", + "# 2. Host Port Mapping\n", + "# This is the port on your local machine that will connect to the container's port 5432.\n", + "POSTGRES_PORT= os.getenv(\"POSTGRES_PORT\")\n", + "\n", + "DATABASE_URL = f\"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/biocirv_db\"\n", + "engine = create_engine(DATABASE_URL)\n", + "print(f\"Connected to database.\")\n", + "\n", + "primary_ag_product = pd.read_sql(\"SELECT * FROM primary_ag_product;\", con=engine)\n", + "\n", + "#reorders columns so id and name are first\n", + "cols = ['id', 'name'] + [c for c in primary_ag_product.columns if c not in ['id', 'name']]\n", + "\n", + "primary_ag_product = primary_ag_product[[*cols]]\n", + "\n", + "primary_ag_product\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
20:08:34.613 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:08:34.613 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:08:36.203 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:08:36.203 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:08:36.211 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:08:36.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'Resource' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 20\u001b[39m\n\u001b[32m 10\u001b[39m df.replace(\u001b[33mr\u001b[39m\u001b[33m'\u001b[39m\u001b[33m^\u001b[39m\u001b[33m\\\u001b[39m\u001b[33ms*$\u001b[39m\u001b[33m'\u001b[39m, np.nan, regex=\u001b[38;5;28;01mTrue\u001b[39;00m, inplace=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# These are columns that need to be normalized, AKA replaced with IDs.\u001b[39;00m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m# This is a mapping that has first, what it is called in pandas \"resource\"\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# then, the SQLAlchemy model \"Resource\", and then what it is called in the\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;66;03m# database \"name\"\u001b[39;00m\n\u001b[32m 19\u001b[39m NORMALIZE_COLUMNS = {\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mresource\u001b[39m\u001b[33m\"\u001b[39m: (\u001b[43mResource\u001b[49m, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 21\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mprepared_sample\u001b[39m\u001b[33m\"\u001b[39m: (PreparedSample, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 22\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mpreparation_method\u001b[39m\u001b[33m\"\u001b[39m: (PreparationMethod, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 23\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m\"\u001b[39m: (Parameter, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 24\u001b[39m \u001b[33m\"\u001b[39m\u001b[33munit\u001b[39m\u001b[33m\"\u001b[39m: (Unit, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 25\u001b[39m \u001b[33m\"\u001b[39m\u001b[33manalyst_email\u001b[39m\u001b[33m\"\u001b[39m: (Contact, \u001b[33m\"\u001b[39m\u001b[33memail\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 26\u001b[39m \u001b[33m\"\u001b[39m\u001b[33manalysis_type\u001b[39m\u001b[33m\"\u001b[39m: (AnalysisType, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 27\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mprimary_ag_product\u001b[39m\u001b[33m\"\u001b[39m: (PrimaryAgProduct, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 28\u001b[39m }\n\u001b[32m 30\u001b[39m df_normalized = df.copy()\n\u001b[32m 32\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Session(engine) \u001b[38;5;28;01mas\u001b[39;00m db:\n", + "\u001b[31mNameError\u001b[39m: name 'Resource' is not defined" + ] + } + ], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cmp_uuid_033record_idprepared_sampleresourcepreparation_methodstorage_condexper_abbrevrepl_norepl_idparameter...unitcreated_atupdated_atqc_resultnoteanalysis_typeequipmentraw_data_urlanalyst_emailupload_status
03ee2993d-86e3-1f16-c7ea-f8d555e114(85)e114oak-tmpm01o(85)tomato pomaceoven dryrt vacuum sealedcmp04xk1.0cmp04xk(85)1glucan...% dry weight2025-01-23 09:00:01NaTpassNaNcompositional analysisNaNNaNxkang2@lbl.govready
146878ef9-1226-22a0-d5d8-cf65e241cb(85)41cboak-tmpm01o(85)tomato pomaceoven dryrt vacuum sealedcmp04xk2.0cmp04xk(85)2glucan...% dry weight2025-01-23 09:00:16NaTpassNaNcompositional analysisNaNNaNxkang2@lbl.govready
276a7a2f4-c4e4-e60f-1187-dec6e02246(85)2246oak-tmpm01o(85)tomato pomaceoven dryrt vacuum sealedcmp04xk3.0cmp04xk(85)3glucan...% dry weight2025-01-23 09:00:31NaTpassNaNcompositional analysisNaNNaNxkang2@lbl.govready
37a136832-286b-07cb-62de-acf52f9311(85)9311oak-tmpm01o(85)tomato pomaceoven dryrt vacuum sealedcmp04xk1.0cmp04xk(85)1glucose...% dry weight2025-01-23 09:00:46NaTpassNaNcompositional analysisNaNNaNxkang2@lbl.govready
4b709ecee-f9a6-a55d-a59e-93b7b863d7(85)63d7oak-tmpm01o(85)tomato pomaceoven dryrt vacuum sealedcmp04xk2.0cmp04xk(85)2glucose...% dry weight2025-01-23 09:01:01NaTpassNaNcompositional analysisNaNNaNxkang2@lbl.govready
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " cmp_uuid_033 record_id prepared_sample \\\n", + "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 oak-tmpm01o(85) \n", + "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb oak-tmpm01o(85) \n", + "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 oak-tmpm01o(85) \n", + "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 oak-tmpm01o(85) \n", + "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 oak-tmpm01o(85) \n", + "\n", + " resource preparation_method storage_cond exper_abbrev repl_no \\\n", + "0 tomato pomace oven dry rt vacuum sealed cmp04xk 1.0 \n", + "1 tomato pomace oven dry rt vacuum sealed cmp04xk 2.0 \n", + "2 tomato pomace oven dry rt vacuum sealed cmp04xk 3.0 \n", + "3 tomato pomace oven dry rt vacuum sealed cmp04xk 1.0 \n", + "4 tomato pomace oven dry rt vacuum sealed cmp04xk 2.0 \n", + "\n", + " repl_id parameter ... unit created_at updated_at \\\n", + "0 cmp04xk(85)1 glucan ... % dry weight 2025-01-23 09:00:01 NaT \n", + "1 cmp04xk(85)2 glucan ... % dry weight 2025-01-23 09:00:16 NaT \n", + "2 cmp04xk(85)3 glucan ... % dry weight 2025-01-23 09:00:31 NaT \n", + "3 cmp04xk(85)1 glucose ... % dry weight 2025-01-23 09:00:46 NaT \n", + "4 cmp04xk(85)2 glucose ... % dry weight 2025-01-23 09:01:01 NaT \n", + "\n", + " qc_result note analysis_type equipment raw_data_url \\\n", + "0 pass NaN compositional analysis NaN NaN \n", + "1 pass NaN compositional analysis NaN NaN \n", + "2 pass NaN compositional analysis NaN NaN \n", + "3 pass NaN compositional analysis NaN NaN \n", + "4 pass NaN compositional analysis NaN NaN \n", + "\n", + " analyst_email upload_status \n", + "0 xkang2@lbl.gov ready \n", + "1 xkang2@lbl.gov ready \n", + "2 xkang2@lbl.gov ready \n", + "3 xkang2@lbl.gov ready \n", + "4 xkang2@lbl.gov ready \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataframes = [df, df2, df3]\n", + "\n", + "clean_dataframes = [clean_the_gsheets(df) for df in dataframes]\n", + "\n", + "clean_dataframes[2].head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.orm import Session\n", + "from sqlalchemy import select\n", + "import pandas as pd\n", + "import os\n", + "import sys\n", + "\n", + "# --- project root discovery (unchanged) ---\n", + "path = os.getcwd()\n", + "project_root = None\n", + "while path != os.path.dirname(path):\n", + " if 'pixi.toml' in os.listdir(path):\n", + " project_root = path\n", + " break\n", + " path = os.path.dirname(path)\n", + "\n", + "if not project_root:\n", + " raise FileNotFoundError(\"Could not find project root containing 'pixi.toml'.\")\n", + "\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# --- imports ---\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import PrimaryAgProduct\n", + "\n", + "# --- query + dataframe ---\n", + "with Session(engine) as db:\n", + " stmt = select(*PrimaryAgProduct.__table__.columns)\n", + " rows = db.execute(stmt).mappings().all()\n", + "\n", + "df = pd.DataFrame(rows)\n", + "\n", + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "summary_stats = clean_dataframes[0].\\\n", + " groupby(['resource', 'parameter'])['value'].\\\n", + " agg(['mean', 'median', 'min', 'max', 'std', 'count'])\n", + "\n", + "summary_stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "clean_dataframes[0][['resource', 'parameter', 'value', 'unit']].\\\n", + " groupby(['resource', 'parameter', 'unit'], as_index=False).\\\n", + " agg({'value': 'mean'}).\\\n", + " query('value > 30').\\\n", + " sort_values(by='value', ascending=False).\\\n", + " round({'value': 1})\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "ename": "KeyError", + "evalue": "'parameter'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:3812\u001b[39m, in \u001b[36mIndex.get_loc\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 3811\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m3812\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 3813\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/index.pyx:167\u001b[39m, in \u001b[36mpandas._libs.index.IndexEngine.get_loc\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/index.pyx:196\u001b[39m, in \u001b[36mpandas._libs.index.IndexEngine.get_loc\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/hashtable_class_helper.pxi:7088\u001b[39m, in \u001b[36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/hashtable_class_helper.pxi:7096\u001b[39m, in \u001b[36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[31mKeyError\u001b[39m: 'parameter'", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 6\u001b[39m df.loc[df[\u001b[33m'\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m'\u001b[39m].isin(list_of_param), \u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mIn list\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[43mis_it_volatile_solids\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 11\u001b[39m df[[\u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m'\u001b[39m]]\n", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 6\u001b[39m, in \u001b[36mis_it_volatile_solids\u001b[39m\u001b[34m(df)\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mis_it_volatile_solids\u001b[39m(df):\n\u001b[32m 4\u001b[39m df[\u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mVS\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m df.loc[\u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mparameter\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m.isin(list_of_param), \u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mIn list\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\frame.py:4113\u001b[39m, in \u001b[36mDataFrame.__getitem__\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 4111\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.columns.nlevels > \u001b[32m1\u001b[39m:\n\u001b[32m 4112\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._getitem_multilevel(key)\n\u001b[32m-> \u001b[39m\u001b[32m4113\u001b[39m indexer = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4114\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[32m 4115\u001b[39m indexer = [indexer]\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:3819\u001b[39m, in \u001b[36mIndex.get_loc\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 3814\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(casted_key, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[32m 3815\u001b[39m \u001b[38;5;28misinstance\u001b[39m(casted_key, abc.Iterable)\n\u001b[32m 3816\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(x, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m casted_key)\n\u001b[32m 3817\u001b[39m ):\n\u001b[32m 3818\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[32m-> \u001b[39m\u001b[32m3819\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01merr\u001b[39;00m\n\u001b[32m 3820\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[32m 3821\u001b[39m \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[32m 3822\u001b[39m \u001b[38;5;66;03m# InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[32m 3823\u001b[39m \u001b[38;5;66;03m# the TypeError.\u001b[39;00m\n\u001b[32m 3824\u001b[39m \u001b[38;5;28mself\u001b[39m._check_indexing_error(key)\n", + "\u001b[31mKeyError\u001b[39m: 'parameter'" + ] + } + ], + "source": [ + "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", + "\n", + "def is_it_volatile_solids(df):\n", + " df['check'] = \"VS\"\n", + "\n", + " df.loc[df['parameter'].isin(list_of_param), 'check'] = \"In list\"\n", + " return df\n", + "\n", + "is_it_volatile_solids(df)\n", + "\n", + "df[['check', 'parameter']]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#This defines a function to calculate the square root of the 'value' column in a DataFrame\n", + "def sqrtvalue(df):\n", + " df = df.assign(sqrtvalue = df['value'] ** 0.5)\n", + " return df\n", + "\n", + "#List comprehension to apply sqrtvalue to each DataFrame\n", + "clean_rooted_df = [sqrtvalue(df) for df in clean_dataframes]\n", + "\n", + "# Display the head of the third DataFrame\n", + "clean_rooted_df[2].head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cmpana_raw = cmpana.extract(project_root=project_root)\n", + "\n", + "cmpana_raw.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
20:10:08.640 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:10:08.640 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:10:10.346 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:10:10.346 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:10:10.356 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:10:10.356 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" + ] + }, + { + "ename": "TypeError", + "evalue": "replace_name_with_id_df() got an unexpected keyword argument 'name_column_name'. Did you mean 'id_column_name'?", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m#this replaces the names with IDs\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Session(engine) \u001b[38;5;28;01mas\u001b[39;00m db:\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m parameter_ids = \u001b[43mreplace_name_with_id_df\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 17\u001b[39m \u001b[43m \u001b[49m\u001b[43mdb\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdb\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 18\u001b[39m \u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtest_df\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 19\u001b[39m \u001b[43m \u001b[49m\u001b[43mref_model\u001b[49m\u001b[43m=\u001b[49m\u001b[43mParameter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 20\u001b[39m \u001b[43m \u001b[49m\u001b[43mname_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# column in df + table\u001b[39;49;00m\n\u001b[32m 21\u001b[39m \u001b[43m \u001b[49m\u001b[43mid_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mid\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# PK column in table\u001b[39;49;00m\n\u001b[32m 22\u001b[39m \u001b[43m \u001b[49m\u001b[43mfinal_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mparameter_id\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\n\u001b[32m 23\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 25\u001b[39m \u001b[38;5;66;03m##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \u001b[39;00m\n\u001b[32m 26\u001b[39m \u001b[38;5;66;03m#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\u001b[39;00m\n\u001b[32m 27\u001b[39m \n\u001b[32m 28\u001b[39m \u001b[38;5;66;03m#resource_id_mapping\u001b[39;00m\n", + "\u001b[31mTypeError\u001b[39m: replace_name_with_id_df() got an unexpected keyword argument 'name_column_name'. Did you mean 'id_column_name'?" + ] + } + ], + "source": [ + "from sqlalchemy.orm import Session\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", + "from src.ca_biositing.datamodels.ca_biositing.datamodels.schemas.generated.ca_biositing import *\n", + "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.name_id_swap import (\n", + " replace_name_with_id_df,\n", + ") \n", + "\n", + "#This extractst the raw proximate data\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "#this cleans the names to lowercase and parses data into a standard format. Also renames the column to match with what will be in the database\n", + "test_df = clean_the_gsheets(df).rename(columns={'parameter': 'name'})\n", + "\n", + "#this replaces the names with IDs\n", + "with Session(engine) as db:\n", + " parameter_ids = replace_name_with_id_df(\n", + " db=db,\n", + " df=test_df,\n", + " ref_model=Parameter,\n", + " name_column_name=\"name\", # column in df + table\n", + " id_column_name=\"id\", # PK column in table\n", + " final_column_name=\"parameter_id\"\n", + " )\n", + "\n", + "##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \n", + "#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\n", + "\n", + "#resource_id_mapping\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parameter_ids\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from prefect import get_run_logger\n", + "\n", + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "resource['name'] = resource['name'].str.lower()\n", + "\n", + "resource['name'] = resource['name'].replace('', np.nan)\n", + "resource.dropna(subset=['name'], inplace=True)\n", + "\n", + "\n", + "resource\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "resource = pd.read_sql(\"SELECT id, name FROM resource\", con=engine)\n", + "\n", + "#this converts the entire dataframe to lowercase\n", + "df = df.map(lambda x: x.lower() if isinstance(x, str) else x)\n", + "\n", + "print(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "field_sample = pd.read_sql_query(\"SELECT * FROM field_sample\", con=engine)\n", + "\n", + "field_sample = field_sample[['id', 'name', 'resource_id']]\n", + "\n", + "field_sample" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is a get_or_create type module for data normalization.\n", + "\n", + "# Extract a df from a gsheet\n", + "df = cmpana.extract(project_root=project_root)\n", + "\n", + "# Cleans the df names and coerces data types\n", + "df = clean_the_gsheets(df)\n", + "\n", + "# Replace empty strings with NaN\n", + "df.replace(r'^\\s*$', np.nan, regex=True, inplace=True)\n", + "\n", + "\n", + "# These are columns that need to be normalized, AKA replaced with IDs.\n", + "# This is a mapping that has first, what it is called in pandas \"resource\"\n", + "# then, the SQLAlchemy model \"Resource\", and then what it is called in the\n", + "# database \"name\"\n", + "\n", + "\n", + "NORMALIZE_COLUMNS = {\n", + " \"resource\": (Resource, \"name\"),\n", + " \"prepared_sample\": (PreparedSample, \"name\"),\n", + " \"preparation_method\": (PreparationMethod, \"name\"),\n", + " \"parameter\": (Parameter, \"name\"),\n", + " \"unit\": (Unit, \"name\"),\n", + " \"analyst_email\": (Contact, \"email\"),\n", + " \"analysis_type\": (AnalysisType, \"name\"),\n", + " \"primary_ag_product\": (PrimaryAgProduct, \"name\")\n", + "}\n", + "\n", + "df_normalized = df.copy()\n", + "\n", + "with Session(engine) as db:\n", + " for df_col, (model, model_name_attr) in NORMALIZE_COLUMNS.items():\n", + " if df_col not in df_normalized.columns:\n", + " continue\n", + "\n", + " df_normalized = replace_name_with_id_df(\n", + " db=db,\n", + " df=df_normalized,\n", + " ref_model=model,\n", + " df_name_column=df_col,\n", + " model_name_attr=model_name_attr,\n", + " id_column_name=\"id\",\n", + " final_column_name=f\"{df_col}_id\",\n", + " )\n", + "\n", + " db.commit()\n", + "\n", + "df_normalized.head()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py index a24e92a0..c5349511 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/engine.py @@ -1,6 +1,32 @@ from sqlmodel import create_engine, Session +import os +from dotenv import load_dotenv -DATABASE_URL = "postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db" +#This module queries the db via the ORM + +# Get the root +path = os.getcwd() +project_root = None +while path != os.path.dirname(path): # Stop at the filesystem root + if 'pixi.toml' in os.listdir(path): + project_root = path + break + path = os.path.dirname(path) + +load_dotenv(dotenv_path=project_root + "\\resources\\docker\\.env") + +# Database Connection +POSTGRES_USER = os.getenv("POSTGRES_USER") +POSTGRES_PASSWORD= os.getenv("POSTGRES_PASSWORD") + +# 2. Host Port Mapping +# This is the port on your local machine that will connect to the container's port 5432. +POSTGRES_PORT= os.getenv("POSTGRES_PORT") + +DATABASE_URL = f"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/biocirv_db" + +# old: +# DATABASE_URL = "postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db" engine = create_engine(DATABASE_URL) db_session = Session(engine) diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb index beb63faf..3c8228d6 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/etl_notebook.ipynb @@ -20,18 +20,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2026-01-06 10:39:39,502 - INFO - Project root '/Users/pjsmitty301/ca-biositing' is already in sys.path\n", - "2026-01-06 10:39:39,503 - INFO - Successfully imported all project modules.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import sys\n", @@ -89,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -240,1483 +231,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2026-01-06 10:53:39,195 - INFO - Starting data extraction...\n", - "2026-01-06 10:53:39,212 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:53:39,266 - INFO - Extracting raw data from '03.1-Proximate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:53:41,468 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:53:41,471 - INFO - Finished in state Completed()\n", - "2026-01-06 10:53:41,486 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:53:41,497 - INFO - Extracting raw data from '03.7-Ultimate' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:53:42,816 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:53:42,820 - INFO - Finished in state Completed()\n", - "2026-01-06 10:53:42,840 - INFO - HTTP Request: GET http://127.0.0.1:4200/api/admin/version \"HTTP/1.1 200 OK\"\n", - "2026-01-06 10:53:42,853 - INFO - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n", - "2026-01-06 10:53:44,427 - INFO - Successfully extracted raw data.\n", - "2026-01-06 10:53:44,430 - INFO - Finished in state Completed()\n", - "2026-01-06 10:53:44,434 - INFO - Data extraction complete.\n", - "2026-01-06 10:53:44,434 - INFO - Starting data cleaning...\n", - "2026-01-06 10:53:44,435 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:53:44,438 - INFO - Dropped 0 rows with missing values.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", - "2026-01-06 10:53:44,450 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", - " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:53:44,455 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:53:44,456 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:53:44,457 - INFO - Dropped 0 rows with missing values.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", - "2026-01-06 10:53:44,461 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", - " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:53:44,467 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:53:44,468 - INFO - Starting DataFrame cleaning process.\n", - "2026-01-06 10:53:44,470 - INFO - Dropped 0 rows with missing values.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:40: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " df_cleaned = df_cleaned.replace(r'^\\s*$', np.nan, regex=True)\n", - "2026-01-06 10:53:44,480 - INFO - Successfully cleaned DataFrame.\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_82269/2660420268.py:48: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", - " df_cleaned = df_cleaned.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "2026-01-06 10:53:44,487 - INFO - Converted all string data to lowercase.\n", - "2026-01-06 10:53:44,492 - INFO - Data cleaning complete.\n", - "2026-01-06 10:53:44,497 - INFO - Starting data normalization...\n", - "2026-01-06 10:53:44,498 - INFO - Starting normalization process for 3 dataframes.\n", - "2026-01-06 10:53:44,498 - INFO - Processing DataFrame #1 with 1030 rows.\n", - "2026-01-06 10:53:44,500 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:53:44,526 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 46 null values.\n", - "2026-01-06 10:53:44,527 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:53:44,531 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 22 null values.\n", - "2026-01-06 10:53:44,531 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:53:44,533 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 22 null values.\n", - "2026-01-06 10:53:44,534 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:53:44,536 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 22 null values.\n", - "2026-01-06 10:53:44,537 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:53:44,539 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:53:44,539 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-06 10:53:44,542 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 550 null values.\n", - "2026-01-06 10:53:44,543 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:53:44,546 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 550 null values.\n", - "2026-01-06 10:53:44,546 - WARNING - Column 'primary_ag_product' not in DataFrame #1. Skipping normalization for this column.\n", - "2026-01-06 10:53:44,546 - INFO - Finished processing DataFrame #1.\n", - "2026-01-06 10:53:44,546 - INFO - Processing DataFrame #2 with 64 rows.\n", - "2026-01-06 10:53:44,547 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:53:44,550 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-06 10:53:44,550 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:53:44,552 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-06 10:53:44,552 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:53:44,554 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-06 10:53:44,554 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:53:44,556 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-06 10:53:44,557 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:53:44,558 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:53:44,559 - INFO - Skipping normalization for column 'analyst_email' as it contains only null values.\n", - "2026-01-06 10:53:44,559 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:53:44,561 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-06 10:53:44,561 - WARNING - Column 'primary_ag_product' not in DataFrame #2. Skipping normalization for this column.\n", - "2026-01-06 10:53:44,561 - INFO - Finished processing DataFrame #2.\n", - "2026-01-06 10:53:44,562 - INFO - Processing DataFrame #3 with 390 rows.\n", - "2026-01-06 10:53:44,562 - INFO - Normalizing column 'resource' using model 'Resource'.\n", - "2026-01-06 10:53:44,567 - INFO - Successfully normalized 'resource'. New column 'resource_id' contains 0 null values.\n", - "2026-01-06 10:53:44,569 - INFO - Normalizing column 'prepared_sample' using model 'PreparedSample'.\n", - "2026-01-06 10:53:44,580 - INFO - Successfully normalized 'prepared_sample'. New column 'prepared_sample_id' contains 0 null values.\n", - "2026-01-06 10:53:44,580 - INFO - Normalizing column 'preparation_method' using model 'PreparationMethod'.\n", - "2026-01-06 10:53:44,592 - INFO - Successfully normalized 'preparation_method'. New column 'preparation_method_id' contains 0 null values.\n", - "2026-01-06 10:53:44,594 - INFO - Normalizing column 'parameter' using model 'Parameter'.\n", - "2026-01-06 10:53:44,597 - INFO - Successfully normalized 'parameter'. New column 'parameter_id' contains 0 null values.\n", - "2026-01-06 10:53:44,597 - INFO - Normalizing column 'unit' using model 'Unit'.\n", - "2026-01-06 10:53:44,598 - INFO - Successfully normalized 'unit'. New column 'unit_id' contains 0 null values.\n", - "2026-01-06 10:53:44,599 - INFO - Normalizing column 'analyst_email' using model 'Contact'.\n", - "2026-01-06 10:53:44,606 - INFO - Successfully normalized 'analyst_email'. New column 'analyst_email_id' contains 0 null values.\n", - "2026-01-06 10:53:44,607 - INFO - Normalizing column 'analysis_type' using model 'AnalysisType'.\n", - "2026-01-06 10:53:44,610 - INFO - Successfully normalized 'analysis_type'. New column 'analysis_type_id' contains 0 null values.\n", - "2026-01-06 10:53:44,610 - WARNING - Column 'primary_ag_product' not in DataFrame #3. Skipping normalization for this column.\n", - "2026-01-06 10:53:44,610 - INFO - Finished processing DataFrame #3.\n", - "2026-01-06 10:53:44,611 - INFO - Committing database session.\n", - "2026-01-06 10:53:44,612 - INFO - Database commit successful.\n", - "2026-01-06 10:53:44,612 - INFO - Data normalization complete.\n", - "2026-01-06 10:53:44,613 - INFO - Displaying results of normalization...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- Normalized DataFrame 1 ---\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "prox_uuid_031", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "source_codename", - "rawType": "object", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "float64", - "type": "float" - }, - { - "name": "repl_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "float64", - "type": "float" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "object", - "type": "string" - }, - { - "name": "upload_status", - "rawType": "object", - "type": "unknown" - }, - { - "name": "note", - "rawType": "float64", - "type": "float" - }, - { - "name": "resource_id", - "rawType": "float64", - "type": "float" - }, - { - "name": "prepared_sample_id", - "rawType": "float64", - "type": "float" - }, - { - "name": "preparation_method_id", - "rawType": "float64", - "type": "float" - }, - { - "name": "parameter_id", - "rawType": "float64", - "type": "float" - }, - { - "name": "unit_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "analyst_email_id", - "rawType": "float64", - "type": "float" - }, - { - "name": "analysis_type_id", - "rawType": "float64", - "type": "float" - } - ], - "ref": "bcafe3ce-a742-48cc-9fb2-da62f6c91695", - "rows": [ - [ - "0", - "d7965110-407f-e356-d41d-b3b9a2b7b7", - "(73)b7b7", - "oakleaf", - "4c", - "prox01xk", - "1.0", - "prox01xk(73)1", - "61.849998474121094", - "2024-10-02 10:31:01", - null, - "pass", - "not ready", - null, - "5.0", - "109.0", - "7.0", - "3.0", - "1", - "1.0", - "1.0" - ], - [ - "1", - "c8fea984-2e9a-8def-55fb-1a9d7d9ba8", - "(73)9ba8", - "oakleaf", - "4c", - "prox01xk", - "2.0", - "prox01xk(73)2", - "63.209999084472656", - "2024-10-02 10:31:31", - null, - "pass", - "ready", - null, - "5.0", - "109.0", - "7.0", - "3.0", - "1", - "1.0", - "1.0" - ], - [ - "2", - "df304d5d-3a85-4881-7142-6d4e5f957d", - "(73)957d", - "oakleaf", - "4c", - "prox01xk", - "3.0", - "prox01xk(73)3", - "63.27000045776367", - "2024-10-02 10:32:01", - null, - "pass", - "imported", - null, - "5.0", - "109.0", - "7.0", - "3.0", - "1", - "1.0", - "1.0" - ], - [ - "3", - "01c6c5be-cea6-54af-3924-b0bad69335", - "(73)9335", - "oakleaf", - "4c", - "prox01xk", - "1.0", - "prox01xk(73)1", - "0.6899999976158142", - "2024-10-03 10:31:01", - null, - "pass", - "import failed", - null, - "5.0", - "109.0", - "7.0", - "4.0", - "1", - "1.0", - "1.0" - ], - [ - "4", - "126745c7-dd41-2f6d-0dc5-28dbca415f", - "(73)415f", - "oakleaf", - "4c", - "prox01xk", - "2.0", - "prox01xk(73)2", - "0.8899999856948853", - "2024-10-03 10:31:31", - null, - "pass", - null, - null, - "5.0", - "109.0", - "7.0", - "4.0", - "1", - "1.0", - "1.0" - ] - ], - "shape": { - "columns": 20, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
prox_uuid_031record_idsource_codenamestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_resultupload_statusnoteresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
0d7965110-407f-e356-d41d-b3b9a2b7b7(73)b7b7oakleaf4cprox01xk1.0prox01xk(73)161.8499982024-10-02 10:31:01NaTpassnot readyNaN5.0109.07.03.011.01.0
1c8fea984-2e9a-8def-55fb-1a9d7d9ba8(73)9ba8oakleaf4cprox01xk2.0prox01xk(73)263.2099992024-10-02 10:31:31NaTpassreadyNaN5.0109.07.03.011.01.0
2df304d5d-3a85-4881-7142-6d4e5f957d(73)957doakleaf4cprox01xk3.0prox01xk(73)363.2700002024-10-02 10:32:01NaTpassimportedNaN5.0109.07.03.011.01.0
301c6c5be-cea6-54af-3924-b0bad69335(73)9335oakleaf4cprox01xk1.0prox01xk(73)10.6900002024-10-03 10:31:01NaTpassimport failedNaN5.0109.07.04.011.01.0
4126745c7-dd41-2f6d-0dc5-28dbca415f(73)415foakleaf4cprox01xk2.0prox01xk(73)20.8900002024-10-03 10:31:31NaTpass<NA>NaN5.0109.07.04.011.01.0
\n", - "
" - ], - "text/plain": [ - " prox_uuid_031 record_id source_codename storage_cond \\\n", - "0 d7965110-407f-e356-d41d-b3b9a2b7b7 (73)b7b7 oakleaf 4c \n", - "1 c8fea984-2e9a-8def-55fb-1a9d7d9ba8 (73)9ba8 oakleaf 4c \n", - "2 df304d5d-3a85-4881-7142-6d4e5f957d (73)957d oakleaf 4c \n", - "3 01c6c5be-cea6-54af-3924-b0bad69335 (73)9335 oakleaf 4c \n", - "4 126745c7-dd41-2f6d-0dc5-28dbca415f (73)415f oakleaf 4c \n", - "\n", - " exper_abbrev repl_no repl_id value created_at \\\n", - "0 prox01xk 1.0 prox01xk(73)1 61.849998 2024-10-02 10:31:01 \n", - "1 prox01xk 2.0 prox01xk(73)2 63.209999 2024-10-02 10:31:31 \n", - "2 prox01xk 3.0 prox01xk(73)3 63.270000 2024-10-02 10:32:01 \n", - "3 prox01xk 1.0 prox01xk(73)1 0.690000 2024-10-03 10:31:01 \n", - "4 prox01xk 2.0 prox01xk(73)2 0.890000 2024-10-03 10:31:31 \n", - "\n", - " updated_at qc_result upload_status note resource_id prepared_sample_id \\\n", - "0 NaT pass not ready NaN 5.0 109.0 \n", - "1 NaT pass ready NaN 5.0 109.0 \n", - "2 NaT pass imported NaN 5.0 109.0 \n", - "3 NaT pass import failed NaN 5.0 109.0 \n", - "4 NaT pass NaN 5.0 109.0 \n", - "\n", - " preparation_method_id parameter_id unit_id analyst_email_id \\\n", - "0 7.0 3.0 1 1.0 \n", - "1 7.0 3.0 1 1.0 \n", - "2 7.0 3.0 1 1.0 \n", - "3 7.0 4.0 1 1.0 \n", - "4 7.0 4.0 1 1.0 \n", - "\n", - " analysis_type_id \n", - "0 1.0 \n", - "1 1.0 \n", - "2 1.0 \n", - "3 1.0 \n", - "4 1.0 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- Normalized DataFrame 2 ---\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "ult_uuid_037", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "ult_sample_name", - "rawType": "object", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "int64", - "type": "integer" - }, - { - "name": "repl_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "float64", - "type": "float" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "string" - }, - { - "name": "equipment", - "rawType": "float64", - "type": "float" - }, - { - "name": "raw_data_url", - "rawType": "float64", - "type": "float" - }, - { - "name": "analyst_email", - "rawType": "float64", - "type": "float" - }, - { - "name": "upload_status", - "rawType": "float64", - "type": "float" - }, - { - "name": "resource_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "prepared_sample_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "preparation_method_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "parameter_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "unit_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "analysis_type_id", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "d0538baa-0f16-4d3e-992f-e11d99faffd6", - "rows": [ - [ - "0", - "421617a5-e50e-9642-2974-b3275fe822", - ")u22e822", - "hum-almhu023km2(15)u22", - "rt vacuum sealed", - "ult26kh", - "1", - "ult26kh(15)1", - "88.5999984741211", - null, - null, - "pass", - "1", - null, - null, - null, - null, - "26", - "80", - "6", - "8", - "3", - "2" - ], - [ - "1", - "7e7919c2-5db4-6bef-75e2-7e51321200", - ")u001200", - "hum-almhu023km2(15)u00", - "rt vacuum sealed", - "ult26kh", - "1", - "ult26kh(15)1", - null, - null, - null, - "fail", - " 1 dup", - null, - null, - null, - null, - "26", - "80", - "6", - "8", - "3", - "2" - ], - [ - "2", - "3aa85881-1185-642f-c44b-41ad2275d2", - ")ud275d2", - "hum-almsh022km2(13)ud2", - "rt vacuum sealed", - "ult26kh", - "1", - "ult26kh(13)1", - "81.30000305175781", - null, - null, - "pass", - "2", - null, - null, - null, - null, - "10", - "96", - "6", - "8", - "3", - "2" - ], - [ - "3", - "fa418804-6c4f-4c90-d78f-84d7df54d3", - ")ud354d3", - "ene-wash017okm2(82)ud3", - "rt vacuum sealed", - "ult26kh", - "1", - "ult26kh(82)1", - "92.19999694824219", - null, - null, - "pass", - "3", - null, - null, - null, - null, - "11", - "91", - "9", - "8", - "3", - "2" - ], - [ - "4", - "6fdacbfc-7e0b-473b-444f-85b7650267", - ")u670267", - "ebo-gppm010okm2(1b)u67", - "rt vacuum sealed", - "ult26kh", - "1", - "ult26kh(1b)1", - "93.80000305175781", - null, - null, - "pass", - "4", - null, - null, - null, - null, - "20", - "153", - "9", - "8", - "3", - "2" - ] - ], - "shape": { - "columns": 22, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ult_uuid_037record_idult_sample_namestorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_at...equipmentraw_data_urlanalyst_emailupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalysis_type_id
0421617a5-e50e-9642-2974-b3275fe822)u22e822hum-almhu023km2(15)u22rt vacuum sealedult26kh1ult26kh(15)188.599998NaTNaT...NaNNaNNaNNaN26806832
17e7919c2-5db4-6bef-75e2-7e51321200)u001200hum-almhu023km2(15)u00rt vacuum sealedult26kh1ult26kh(15)1NaNNaTNaT...NaNNaNNaNNaN26806832
23aa85881-1185-642f-c44b-41ad2275d2)ud275d2hum-almsh022km2(13)ud2rt vacuum sealedult26kh1ult26kh(13)181.300003NaTNaT...NaNNaNNaNNaN10966832
3fa418804-6c4f-4c90-d78f-84d7df54d3)ud354d3ene-wash017okm2(82)ud3rt vacuum sealedult26kh1ult26kh(82)192.199997NaTNaT...NaNNaNNaNNaN11919832
46fdacbfc-7e0b-473b-444f-85b7650267)u670267ebo-gppm010okm2(1b)u67rt vacuum sealedult26kh1ult26kh(1b)193.800003NaTNaT...NaNNaNNaNNaN201539832
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " ult_uuid_037 record_id ult_sample_name \\\n", - "0 421617a5-e50e-9642-2974-b3275fe822 )u22e822 hum-almhu023km2(15)u22 \n", - "1 7e7919c2-5db4-6bef-75e2-7e51321200 )u001200 hum-almhu023km2(15)u00 \n", - "2 3aa85881-1185-642f-c44b-41ad2275d2 )ud275d2 hum-almsh022km2(13)ud2 \n", - "3 fa418804-6c4f-4c90-d78f-84d7df54d3 )ud354d3 ene-wash017okm2(82)ud3 \n", - "4 6fdacbfc-7e0b-473b-444f-85b7650267 )u670267 ebo-gppm010okm2(1b)u67 \n", - "\n", - " storage_cond exper_abbrev repl_no repl_id value created_at \\\n", - "0 rt vacuum sealed ult26kh 1 ult26kh(15)1 88.599998 NaT \n", - "1 rt vacuum sealed ult26kh 1 ult26kh(15)1 NaN NaT \n", - "2 rt vacuum sealed ult26kh 1 ult26kh(13)1 81.300003 NaT \n", - "3 rt vacuum sealed ult26kh 1 ult26kh(82)1 92.199997 NaT \n", - "4 rt vacuum sealed ult26kh 1 ult26kh(1b)1 93.800003 NaT \n", - "\n", - " updated_at ... equipment raw_data_url analyst_email upload_status \\\n", - "0 NaT ... NaN NaN NaN NaN \n", - "1 NaT ... NaN NaN NaN NaN \n", - "2 NaT ... NaN NaN NaN NaN \n", - "3 NaT ... NaN NaN NaN NaN \n", - "4 NaT ... NaN NaN NaN NaN \n", - "\n", - " resource_id prepared_sample_id preparation_method_id parameter_id \\\n", - "0 26 80 6 8 \n", - "1 26 80 6 8 \n", - "2 10 96 6 8 \n", - "3 11 91 9 8 \n", - "4 20 153 9 8 \n", - "\n", - " unit_id analysis_type_id \n", - "0 3 2 \n", - "1 3 2 \n", - "2 3 2 \n", - "3 3 2 \n", - "4 3 2 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- Normalized DataFrame 3 ---\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "cmp_uuid_033", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "int64", - "type": "integer" - }, - { - "name": "repl_id", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "float64", - "type": "float" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "float64", - "type": "float" - }, - { - "name": "equipment", - "rawType": "float64", - "type": "float" - }, - { - "name": "raw_data_url", - "rawType": "float64", - "type": "float" - }, - { - "name": "upload_status", - "rawType": "object", - "type": "string" - }, - { - "name": "resource_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "prepared_sample_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "preparation_method_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "parameter_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "unit_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "analyst_email_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "analysis_type_id", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "1f353831-a3bd-4f43-a253-769adc8b66eb", - "rows": [ - [ - "0", - "3ee2993d-86e3-1f16-c7ea-f8d555e114", - "(85)e114", - "rt vacuum sealed", - "cmp04xk", - "1", - "cmp04xk(85)1", - "14.15999984741211", - "2025-01-23 09:00:01", - null, - "pass", - null, - null, - null, - "ready", - "5", - "92", - "8", - "13", - "2", - "1", - "3" - ], - [ - "1", - "46878ef9-1226-22a0-d5d8-cf65e241cb", - "(85)41cb", - "rt vacuum sealed", - "cmp04xk", - "2", - "cmp04xk(85)2", - "14.180000305175781", - "2025-01-23 09:00:16", - null, - "pass", - null, - null, - null, - "ready", - "5", - "92", - "8", - "13", - "2", - "1", - "3" - ], - [ - "2", - "76a7a2f4-c4e4-e60f-1187-dec6e02246", - "(85)2246", - "rt vacuum sealed", - "cmp04xk", - "3", - "cmp04xk(85)3", - "14.119999885559082", - "2025-01-23 09:00:31", - null, - "pass", - null, - null, - null, - "ready", - "5", - "92", - "8", - "13", - "2", - "1", - "3" - ], - [ - "3", - "7a136832-286b-07cb-62de-acf52f9311", - "(85)9311", - "rt vacuum sealed", - "cmp04xk", - "1", - "cmp04xk(85)1", - "15.739999771118164", - "2025-01-23 09:00:46", - null, - "pass", - null, - null, - null, - "ready", - "5", - "92", - "8", - "11", - "2", - "1", - "3" - ], - [ - "4", - "b709ecee-f9a6-a55d-a59e-93b7b863d7", - "(85)63d7", - "rt vacuum sealed", - "cmp04xk", - "2", - "cmp04xk(85)2", - "15.75", - "2025-01-23 09:01:01", - null, - "pass", - null, - null, - null, - "ready", - "5", - "92", - "8", - "11", - "2", - "1", - "3" - ] - ], - "shape": { - "columns": 21, - "rows": 5 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cmp_uuid_033record_idstorage_condexper_abbrevrepl_norepl_idvaluecreated_atupdated_atqc_result...equipmentraw_data_urlupload_statusresource_idprepared_sample_idpreparation_method_idparameter_idunit_idanalyst_email_idanalysis_type_id
03ee2993d-86e3-1f16-c7ea-f8d555e114(85)e114rt vacuum sealedcmp04xk1cmp04xk(85)114.162025-01-23 09:00:01NaTpass...NaNNaNready592813213
146878ef9-1226-22a0-d5d8-cf65e241cb(85)41cbrt vacuum sealedcmp04xk2cmp04xk(85)214.182025-01-23 09:00:16NaTpass...NaNNaNready592813213
276a7a2f4-c4e4-e60f-1187-dec6e02246(85)2246rt vacuum sealedcmp04xk3cmp04xk(85)314.122025-01-23 09:00:31NaTpass...NaNNaNready592813213
37a136832-286b-07cb-62de-acf52f9311(85)9311rt vacuum sealedcmp04xk1cmp04xk(85)115.742025-01-23 09:00:46NaTpass...NaNNaNready592811213
4b709ecee-f9a6-a55d-a59e-93b7b863d7(85)63d7rt vacuum sealedcmp04xk2cmp04xk(85)215.752025-01-23 09:01:01NaTpass...NaNNaNready592811213
\n", - "

5 rows × 21 columns

\n", - "
" - ], - "text/plain": [ - " cmp_uuid_033 record_id storage_cond \\\n", - "0 3ee2993d-86e3-1f16-c7ea-f8d555e114 (85)e114 rt vacuum sealed \n", - "1 46878ef9-1226-22a0-d5d8-cf65e241cb (85)41cb rt vacuum sealed \n", - "2 76a7a2f4-c4e4-e60f-1187-dec6e02246 (85)2246 rt vacuum sealed \n", - "3 7a136832-286b-07cb-62de-acf52f9311 (85)9311 rt vacuum sealed \n", - "4 b709ecee-f9a6-a55d-a59e-93b7b863d7 (85)63d7 rt vacuum sealed \n", - "\n", - " exper_abbrev repl_no repl_id value created_at updated_at \\\n", - "0 cmp04xk 1 cmp04xk(85)1 14.16 2025-01-23 09:00:01 NaT \n", - "1 cmp04xk 2 cmp04xk(85)2 14.18 2025-01-23 09:00:16 NaT \n", - "2 cmp04xk 3 cmp04xk(85)3 14.12 2025-01-23 09:00:31 NaT \n", - "3 cmp04xk 1 cmp04xk(85)1 15.74 2025-01-23 09:00:46 NaT \n", - "4 cmp04xk 2 cmp04xk(85)2 15.75 2025-01-23 09:01:01 NaT \n", - "\n", - " qc_result ... equipment raw_data_url upload_status resource_id \\\n", - "0 pass ... NaN NaN ready 5 \n", - "1 pass ... NaN NaN ready 5 \n", - "2 pass ... NaN NaN ready 5 \n", - "3 pass ... NaN NaN ready 5 \n", - "4 pass ... NaN NaN ready 5 \n", - "\n", - " prepared_sample_id preparation_method_id parameter_id unit_id \\\n", - "0 92 8 13 2 \n", - "1 92 8 13 2 \n", - "2 92 8 13 2 \n", - "3 92 8 11 2 \n", - "4 92 8 11 2 \n", - "\n", - " analyst_email_id analysis_type_id \n", - "0 1 3 \n", - "1 1 3 \n", - "2 1 3 \n", - "3 1 3 \n", - "4 1 3 \n", - "\n", - "[5 rows x 21 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# --- 1. Extraction ---\n", "# In a real Prefect flow, each extraction would be a separate task.\n", @@ -1779,7 +296,7 @@ ], "metadata": { "kernelspec": { - "display_name": "default", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb index bfb3c8c4..b9feba10 100644 --- a/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb +++ b/src/ca_biositing/pipeline/ca_biositing/pipeline/utils/gsheet_extraction_notebook.ipynb @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -118,415 +118,30 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected to database.\n" - ] - }, - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "name", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "unknown" - }, - { - "name": "description", - "rawType": "object", - "type": "unknown" - }, - { - "name": "uri", - "rawType": "object", - "type": "unknown" - } - ], - "ref": "f594ce12-31fc-400e-807f-9ae024f154ba", - "rows": [ - [ - "0", - "1", - "Tomatoes for processing", - null, - null, - null - ], - [ - "1", - "2", - "Grapes", - null, - null, - null - ], - [ - "2", - "3", - "Almonds", - null, - null, - null - ], - [ - "3", - "4", - "Walnuts", - null, - null, - null - ], - [ - "4", - "5", - "Sweet potatoes", - null, - null, - null - ], - [ - "5", - "6", - "Algae", - null, - null, - null - ], - [ - "6", - "7", - "Olives - processing", - null, - null, - null - ], - [ - "7", - "8", - "Corn - all", - null, - null, - null - ], - [ - "8", - "9", - "Hay - alfalfa", - null, - null, - null - ], - [ - "9", - "10", - "Silage - wheat", - null, - null, - null - ], - [ - "10", - "11", - "Rice", - null, - null, - null - ], - [ - "11", - "12", - "Peaches", - null, - null, - null - ], - [ - "12", - "13", - "Potatoes", - null, - null, - null - ], - [ - "13", - "14", - "Cucumbers and pickles", - null, - null, - null - ], - [ - "14", - "15", - "Pistachios", - null, - null, - null - ], - [ - "15", - "16", - "Cotton", - null, - null, - null - ], - [ - "16", - "17", - "Olives - market", - null, - null, - null - ], - [ - "17", - "18", - "", - null, - null, - null - ] - ], - "shape": { - "columns": 5, - "rows": 18 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamenotedescriptionuri
01Tomatoes for processingNoneNoneNone
12GrapesNoneNoneNone
23AlmondsNoneNoneNone
34WalnutsNoneNoneNone
45Sweet potatoesNoneNoneNone
56AlgaeNoneNoneNone
67Olives - processingNoneNoneNone
78Corn - allNoneNoneNone
89Hay - alfalfaNoneNoneNone
910Silage - wheatNoneNoneNone
1011RiceNoneNoneNone
1112PeachesNoneNoneNone
1213PotatoesNoneNoneNone
1314Cucumbers and picklesNoneNoneNone
1415PistachiosNoneNoneNone
1516CottonNoneNoneNone
1617Olives - marketNoneNoneNone
1718NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " id name note description uri\n", - "0 1 Tomatoes for processing None None None\n", - "1 2 Grapes None None None\n", - "2 3 Almonds None None None\n", - "3 4 Walnuts None None None\n", - "4 5 Sweet potatoes None None None\n", - "5 6 Algae None None None\n", - "6 7 Olives - processing None None None\n", - "7 8 Corn - all None None None\n", - "8 9 Hay - alfalfa None None None\n", - "9 10 Silage - wheat None None None\n", - "10 11 Rice None None None\n", - "11 12 Peaches None None None\n", - "12 13 Potatoes None None None\n", - "13 14 Cucumbers and pickles None None None\n", - "14 15 Pistachios None None None\n", - "15 16 Cotton None None None\n", - "16 17 Olives - market None None None\n", - "17 18 None None None" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sqlmodel import Session, select, create_engine\n", "import pandas as pd\n", "import os\n", "import sys\n", + "from dotenv import load_dotenv\n", "\n", "#This module queries the db via the ORM\n", "\n", + "load_dotenv(dotenv_path=project_root + \"\\\\resources\\\\docker\\\\.env\")\n", + "\n", "# Database Connection\n", - "DATABASE_URL = \"postgresql+psycopg2://biocirv_user:biocirv_dev_password@localhost:5432/biocirv_db\"\n", + "\n", + "POSTGRES_USER = os.getenv(\"POSTGRES_USER\")\n", + "POSTGRES_PASSWORD= os.getenv(\"POSTGRES_PASSWORD\")\n", + "\n", + "# 2. Host Port Mapping\n", + "# This is the port on your local machine that will connect to the container's port 5432.\n", + "POSTGRES_PORT= os.getenv(\"POSTGRES_PORT\")\n", + "\n", + "DATABASE_URL = f\"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/biocirv_db\"\n", "engine = create_engine(DATABASE_URL)\n", "print(f\"Connected to database.\")\n", "\n", @@ -542,9 +157,70 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
20:08:34.613 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:08:34.613 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:08:36.203 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:08:36.203 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:08:36.211 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:08:36.211 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'Resource' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 20\u001b[39m\n\u001b[32m 10\u001b[39m df.replace(\u001b[33mr\u001b[39m\u001b[33m'\u001b[39m\u001b[33m^\u001b[39m\u001b[33m\\\u001b[39m\u001b[33ms*$\u001b[39m\u001b[33m'\u001b[39m, np.nan, regex=\u001b[38;5;28;01mTrue\u001b[39;00m, inplace=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# These are columns that need to be normalized, AKA replaced with IDs.\u001b[39;00m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m# This is a mapping that has first, what it is called in pandas \"resource\"\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# then, the SQLAlchemy model \"Resource\", and then what it is called in the\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;66;03m# database \"name\"\u001b[39;00m\n\u001b[32m 19\u001b[39m NORMALIZE_COLUMNS = {\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mresource\u001b[39m\u001b[33m\"\u001b[39m: (\u001b[43mResource\u001b[49m, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 21\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mprepared_sample\u001b[39m\u001b[33m\"\u001b[39m: (PreparedSample, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 22\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mpreparation_method\u001b[39m\u001b[33m\"\u001b[39m: (PreparationMethod, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 23\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m\"\u001b[39m: (Parameter, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 24\u001b[39m \u001b[33m\"\u001b[39m\u001b[33munit\u001b[39m\u001b[33m\"\u001b[39m: (Unit, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 25\u001b[39m \u001b[33m\"\u001b[39m\u001b[33manalyst_email\u001b[39m\u001b[33m\"\u001b[39m: (Contact, \u001b[33m\"\u001b[39m\u001b[33memail\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 26\u001b[39m \u001b[33m\"\u001b[39m\u001b[33manalysis_type\u001b[39m\u001b[33m\"\u001b[39m: (AnalysisType, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m 27\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mprimary_ag_product\u001b[39m\u001b[33m\"\u001b[39m: (PrimaryAgProduct, \u001b[33m\"\u001b[39m\u001b[33mname\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 28\u001b[39m }\n\u001b[32m 30\u001b[39m df_normalized = df.copy()\n\u001b[32m 32\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Session(engine) \u001b[38;5;28;01mas\u001b[39;00m db:\n", + "\u001b[31mNameError\u001b[39m: name 'Resource' is not defined" + ] + } + ], "source": [ "#This is a get_or_create type module for data normalization.\n", "\n", @@ -606,257 +282,20 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", - "/var/folders/2l/qpqn5_6578z142wxn32lbtw00000gn/T/ipykernel_9286/3648773749.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", - " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n" + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" ] }, { "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "cmp_uuid_033", - "rawType": "object", - "type": "string" - }, - { - "name": "record_id", - "rawType": "object", - "type": "string" - }, - { - "name": "prepared_sample", - "rawType": "object", - "type": "string" - }, - { - "name": "resource", - "rawType": "object", - "type": "string" - }, - { - "name": "preparation_method", - "rawType": "object", - "type": "string" - }, - { - "name": "storage_cond", - "rawType": "object", - "type": "string" - }, - { - "name": "exper_abbrev", - "rawType": "object", - "type": "string" - }, - { - "name": "repl_no", - "rawType": "int64", - "type": "integer" - }, - { - "name": "repl_id", - "rawType": "object", - "type": "string" - }, - { - "name": "parameter", - "rawType": "object", - "type": "string" - }, - { - "name": "value", - "rawType": "float64", - "type": "float" - }, - { - "name": "unit", - "rawType": "object", - "type": "string" - }, - { - "name": "created_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "updated_at", - "rawType": "datetime64[ns]", - "type": "datetime" - }, - { - "name": "qc_result", - "rawType": "object", - "type": "string" - }, - { - "name": "note", - "rawType": "object", - "type": "string" - }, - { - "name": "analysis_type", - "rawType": "object", - "type": "string" - }, - { - "name": "equipment", - "rawType": "object", - "type": "string" - }, - { - "name": "raw_data_url", - "rawType": "object", - "type": "string" - }, - { - "name": "analyst_email", - "rawType": "object", - "type": "string" - }, - { - "name": "upload_status", - "rawType": "object", - "type": "string" - } - ], - "ref": "b4e5431c-6847-48e5-894f-3a2387854a2b", - "rows": [ - [ - "0", - "3ee2993d-86e3-1f16-c7ea-f8d555e114", - "(85)e114", - "oak-tmpm01o(85)", - "tomato pomace", - "oven dry", - "rt vacuum sealed", - "cmp04xk", - "1", - "cmp04xk(85)1", - "glucan", - "14.15999984741211", - "% dry weight", - "2025-01-23 09:00:01", - null, - "pass", - "", - "chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "1", - "46878ef9-1226-22a0-d5d8-cf65e241cb", - "(85)41cb", - "oak-tmpm01o(85)", - "tomato pomace", - "oven dry", - "rt vacuum sealed", - "cmp04xk", - "2", - "cmp04xk(85)2", - "glucan", - "14.180000305175781", - "% dry weight", - "2025-01-23 09:00:16", - null, - "pass", - "", - "chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "2", - "76a7a2f4-c4e4-e60f-1187-dec6e02246", - "(85)2246", - "oak-tmpm01o(85)", - "tomato pomace", - "oven dry", - "rt vacuum sealed", - "cmp04xk", - "3", - "cmp04xk(85)3", - "glucan", - "14.119999885559082", - "% dry weight", - "2025-01-23 09:00:31", - null, - "pass", - "", - "chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "3", - "7a136832-286b-07cb-62de-acf52f9311", - "(85)9311", - "oak-tmpm01o(85)", - "tomato pomace", - "oven dry", - "rt vacuum sealed", - "cmp04xk", - "1", - "cmp04xk(85)1", - "glucose", - "15.739999771118164", - "% dry weight", - "2025-01-23 09:00:46", - null, - "pass", - "", - "chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ], - [ - "4", - "b709ecee-f9a6-a55d-a59e-93b7b863d7", - "(85)63d7", - "oak-tmpm01o(85)", - "tomato pomace", - "oven dry", - "rt vacuum sealed", - "cmp04xk", - "2", - "cmp04xk(85)2", - "glucose", - "15.75", - "% dry weight", - "2025-01-23 09:01:01", - null, - "pass", - "", - "chemical composition", - "", - "", - "xkang2@lbl.gov", - "ready" - ] - ], - "shape": { - "columns": 21, - "rows": 5 - } - }, "text/html": [ "
\n", "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
descriptionidnamenoteuri
0None1Tomatoes for processingNoneNone
1None2GrapesNoneNone
2None3AlmondsNoneNone
3None4WalnutsNoneNone
4None5Sweet potatoesNoneNone
\n", - "
" - ], - "text/plain": [ - " description id name note uri\n", - "0 None 1 Tomatoes for processing None None\n", - "1 None 2 Grapes None None\n", - "2 None 3 Almonds None None\n", - "3 None 4 Walnuts None None\n", - "4 None 5 Sweet potatoes None None" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sqlalchemy.orm import Session\n", "from sqlalchemy import select\n", @@ -1315,9 +586,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "KeyError", + "evalue": "'parameter'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:3812\u001b[39m, in \u001b[36mIndex.get_loc\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 3811\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m3812\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 3813\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/index.pyx:167\u001b[39m, in \u001b[36mpandas._libs.index.IndexEngine.get_loc\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/index.pyx:196\u001b[39m, in \u001b[36mpandas._libs.index.IndexEngine.get_loc\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/hashtable_class_helper.pxi:7088\u001b[39m, in \u001b[36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/hashtable_class_helper.pxi:7096\u001b[39m, in \u001b[36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[39m\u001b[34m()\u001b[39m\n", + "\u001b[31mKeyError\u001b[39m: 'parameter'", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 6\u001b[39m df.loc[df[\u001b[33m'\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m'\u001b[39m].isin(list_of_param), \u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mIn list\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[43mis_it_volatile_solids\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 11\u001b[39m df[[\u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mparameter\u001b[39m\u001b[33m'\u001b[39m]]\n", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 6\u001b[39m, in \u001b[36mis_it_volatile_solids\u001b[39m\u001b[34m(df)\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mis_it_volatile_solids\u001b[39m(df):\n\u001b[32m 4\u001b[39m df[\u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mVS\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m df.loc[\u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mparameter\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m.isin(list_of_param), \u001b[33m'\u001b[39m\u001b[33mcheck\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33mIn list\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\frame.py:4113\u001b[39m, in \u001b[36mDataFrame.__getitem__\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 4111\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.columns.nlevels > \u001b[32m1\u001b[39m:\n\u001b[32m 4112\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._getitem_multilevel(key)\n\u001b[32m-> \u001b[39m\u001b[32m4113\u001b[39m indexer = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4114\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[32m 4115\u001b[39m indexer = [indexer]\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\OneDrive\\Documents\\GitHub\\ca-biositing\\.pixi\\envs\\default\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:3819\u001b[39m, in \u001b[36mIndex.get_loc\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 3814\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(casted_key, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[32m 3815\u001b[39m \u001b[38;5;28misinstance\u001b[39m(casted_key, abc.Iterable)\n\u001b[32m 3816\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(x, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m casted_key)\n\u001b[32m 3817\u001b[39m ):\n\u001b[32m 3818\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[32m-> \u001b[39m\u001b[32m3819\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01merr\u001b[39;00m\n\u001b[32m 3820\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[32m 3821\u001b[39m \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[32m 3822\u001b[39m \u001b[38;5;66;03m# InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[32m 3823\u001b[39m \u001b[38;5;66;03m# the TypeError.\u001b[39;00m\n\u001b[32m 3824\u001b[39m \u001b[38;5;28mself\u001b[39m._check_indexing_error(key)\n", + "\u001b[31mKeyError\u001b[39m: 'parameter'" + ] + } + ], "source": [ "list_of_param = (\"Moisture\", \"Total solids\", \"Ash\")\n", "\n", @@ -1365,9 +659,70 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
20:10:08.640 | INFO    | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n",
+       "
\n" + ], + "text/plain": [ + "20:10:08.640 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Extracting raw data from '03.3-CmpAna' in 'Aim 1-Feedstock Collection and Processing Data-BioCirV'...\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:10:10.346 | INFO    | Task run 'extract' - Successfully extracted raw data.\n",
+       "
\n" + ], + "text/plain": [ + "20:10:10.346 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Successfully extracted raw data.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
20:10:10.356 | INFO    | Task run 'extract' - Finished in state Completed()\n",
+       "
\n" + ], + "text/plain": [ + "20:10:10.356 | \u001b[36mINFO\u001b[0m | Task run 'extract' - Finished in state \u001b[32mCompleted\u001b[0m()\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:17: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(lambda s: s.lower() if isinstance(s, str) else s)\n", + "C:\\Users\\Abigail\\AppData\\Local\\Temp\\ipykernel_36096\\4080610521.py:20: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " df.replace(\"\", np.nan, inplace=True)\n" + ] + }, + { + "ename": "TypeError", + "evalue": "replace_name_with_id_df() got an unexpected keyword argument 'name_column_name'. Did you mean 'id_column_name'?", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m#this replaces the names with IDs\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Session(engine) \u001b[38;5;28;01mas\u001b[39;00m db:\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m parameter_ids = \u001b[43mreplace_name_with_id_df\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 17\u001b[39m \u001b[43m \u001b[49m\u001b[43mdb\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdb\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 18\u001b[39m \u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtest_df\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 19\u001b[39m \u001b[43m \u001b[49m\u001b[43mref_model\u001b[49m\u001b[43m=\u001b[49m\u001b[43mParameter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 20\u001b[39m \u001b[43m \u001b[49m\u001b[43mname_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# column in df + table\u001b[39;49;00m\n\u001b[32m 21\u001b[39m \u001b[43m \u001b[49m\u001b[43mid_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mid\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# PK column in table\u001b[39;49;00m\n\u001b[32m 22\u001b[39m \u001b[43m \u001b[49m\u001b[43mfinal_column_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mparameter_id\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\n\u001b[32m 23\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 25\u001b[39m \u001b[38;5;66;03m##I EVENTUALLY WANT SOME LOGS ABOUT HOW MANY WERE ADDED, HOW MANY RETRIEVED, ETC. MAYBE PUT THAT IN THE \u001b[39;00m\n\u001b[32m 26\u001b[39m \u001b[38;5;66;03m#resource_id_mapping = df_with_ids.rename(columns={\"id\": \"resource_id\"})\u001b[39;00m\n\u001b[32m 27\u001b[39m \n\u001b[32m 28\u001b[39m \u001b[38;5;66;03m#resource_id_mapping\u001b[39;00m\n", + "\u001b[31mTypeError\u001b[39m: replace_name_with_id_df() got an unexpected keyword argument 'name_column_name'. Did you mean 'id_column_name'?" + ] + } + ], "source": [ "from sqlalchemy.orm import Session\n", "from src.ca_biositing.pipeline.ca_biositing.pipeline.utils.engine import engine\n", @@ -1522,7 +877,7 @@ ], "metadata": { "kernelspec": { - "display_name": "default", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" },