Added scripts and dependencies for MepRAM RedCAP to OMOP ETL

scripts/Queries.py

@@ -0,0 +1,119 @@
+class ETLQueries:
+    """
+    Class containing the different SQL queries used to interact with OMOP tables for MePRAM ETL
+    """
+
+    load_observation_table_query = """
+    INSERT INTO cdm.observation(
+        person_id, observation_concept_id, observation_date, 
+        observation_type_concept_id, value_as_number, value_as_concept_id, value_source_value, observation_event_id, obs_event_field_concept_id, visit_occurrence_id, observation_source_value
+    ) VALUES(
+        :person_id, :observation_concept_id, :observation_date, 
+        :observation_type_concept_id, :value_as_number, :value_as_concept_id, :value_source_value, :observation_event_id, :obs_event_field_concept_id, :visit_occurrence_id, :observation_source_value
+    ) RETURNING observation_id
+    """
+
+    load_observation_hepatopatia = """
+    INSERT INTO cdm.observation(
+        person_id, observation_concept_id, observation_date, 
+        observation_type_concept_id, value_as_number, value_as_concept_id, qualifier_concept_id, value_source_value, observation_event_id, obs_event_field_concept_id, visit_occurrence_id
+    ) VALUES(
+        :person_id, :observation_concept_id, :observation_date, 
+        :observation_type_concept_id, :value_as_number, :value_as_concept_id, :qualifier_concept_id, :value_source_value, :observation_event_id, :obs_event_field_concept_id, :visit_occurrence_id
+    ) RETURNING observation_id
+    """
+
+    load_condition_table_query = """
+    INSERT INTO cdm.condition_occurrence(
+        person_id, condition_concept_id, condition_start_date, 
+        condition_type_concept_id, visit_occurrence_id, condition_source_value
+        ) VALUES(
+        :person_id, :condition_concept_id, :condition_start_date, 
+        :condition_type_concept_id, :visit_occurrence_id, :condition_source_value
+    ) RETURNING condition_occurrence_id
+    """
+
+    load_condition_foco_query = """
+    INSERT INTO cdm.condition_occurrence(
+        person_id, condition_concept_id, condition_start_date, 
+        condition_type_concept_id, condition_status_concept_id, visit_occurrence_id, condition_source_value
+        ) VALUES(
+        :person_id, :condition_concept_id, :condition_start_date, 
+        :condition_type_concept_id, :condition_status_concept_id, :visit_occurrence_id, :condition_source_value
+    ) RETURNING condition_occurrence_id
+    """
+
+    load_measurement_table_query = """
+    INSERT INTO cdm.measurement(
+        person_id, measurement_concept_id, measurement_date, 
+        measurement_type_concept_id, operator_concept_id, value_as_number, 
+        value_as_concept_id, unit_concept_id, measurement_event_id, meas_event_field_concept_id, visit_occurrence_id, measurement_source_value, value_source_value
+    ) VALUES(
+        :person_id, :measurement_concept_id, :measurement_date, 
+        :measurement_type_concept_id, :operator_concept_id, :value_as_number, 
+        :value_as_concept_id, :unit_concept_id, :measurement_event_id, :meas_event_field_concept_id, :visit_occurrence_id, :measurement_source_value, :value_source_value
+    ) RETURNING measurement_id
+    """
+
+    load_procedure_occurrence_table = """INSERT INTO cdm.procedure_occurrence(person_id, procedure_concept_id, procedure_date, procedure_type_concept_id, visit_occurrence_id, procedure_source_value
+    ) VALUES(:person_id, :procedure_concept_id, :procedure_date, :procedure_type_concept_id, :visit_occurrence_id, :procedure_source_value)"""
+
+    load_episode_table = "INSERT INTO cdm.episode(person_id, episode_concept_id, episode_start_date, episode_type_concept_id, episode_object_concept_id) VALUES(:person_id, :episode_concept_id, :episode_start_date, :episode_type_concept_id, :episode_object_concept_id) RETURNING episode_id"
+
+    load_episode_event_table = "INSERT INTO cdm.episode_event(episode_id, event_id, episode_event_field_concept_id) VALUES(:episode_id, :event_id, :episode_event_field_concept_id)"
+
+    query_drug_exposure_table = """INSERT INTO cdm.drug_exposure(person_id, drug_concept_id, drug_exposure_start_date, drug_exposure_end_date, drug_type_concept_id, visit_occurrence_id, drug_source_value) 
+    VALUES(:person_id, :drug_concept_id, :drug_exposure_start_date, :drug_exposure_end_date, :drug_type_concept_id, :visit_occurrence_id, :drug_source_value) RETURNING drug_exposure_id"""
+
+    load_specimen_table_query = """INSERT INTO cdm.specimen(person_id, specimen_concept_id, specimen_date, specimen_type_concept_id)
+    VALUES(:person_id, :specimen_concept_id, :specimen_date, :specimen_type_concept_id) RETURNING specimen_id"""
+
+    get_fecha_ingreso_query = "SELECT observation_date FROM cdm.observation WHERE person_id = :person_id AND observation_concept_id = :fecha_ingreso_concept_id"
+
+    get_care_site_id_query = "SELECT care_site_id FROM cdm.care_site WHERE care_site_source_value = :care_site_source_value"
+
+    load_person_table_query = """INSERT INTO cdm.person(person_id, gender_concept_id, year_of_birth, birth_datetime, race_concept_id, ethnicity_concept_id, care_site_id, person_source_value, gender_source_value) 
+    VALUES(:person_id, :gender_concept_id, :year_of_birth, :birth_datetime, 0, 0, :care_site_id, :person_source_value, :gender_source_value)"""
+
+    get_visit_occurrence_query = "SELECT visit_occurrence_id FROM cdm.visit_occurrence WHERE person_id = :person_id"
+
+    get_episode_id = "SELECT episode_id FROM cdm.episode where person_id = :person_id"
+
+    get_valid_patients_for_relationship_table = """
+        SELECT DISTINCT po.person_id
+        FROM management.etl_tracking a
+        JOIN management.etl_tracking b ON a.source_person_id = b.source_person_id
+        JOIN management.person_origin po ON a.source_person_id = po.source_person_id
+        JOIN cdm.episode e ON po.person_id = e.person_id
+        WHERE a.form_name = :form_name_1
+        AND b.form_name = :form_name_2
+        AND e.episode_object_concept_id = 432250;
+    """
+    get_condition_occurrence_id_from_episode = """
+        WITH relevant_episode AS (
+            SELECT episode_id
+            FROM cdm.episode
+            WHERE person_id = :person_id
+        )
+        SELECT event_id
+        FROM cdm.episode_event
+        WHERE episode_id IN (SELECT episode_id FROM relevant_episode)
+        AND episode_event_field_concept_id = 1147129;
+    """
+    get_drug_exposure_id_from_episode = """
+        WITH relevant_episode AS (
+            SELECT episode_id
+            FROM cdm.episode
+            WHERE person_id = :person_id
+        )
+        SELECT event_id
+        FROM cdm.episode_event
+        WHERE episode_id IN (SELECT episode_id FROM relevant_episode)
+        AND episode_event_field_concept_id = 1147096;
+    """
+    filter_redcap_query = """
+    UPDATE ETL_management
+    SET quality_check = 'invalid'
+    WHERE source_person_id = :source_person_id
+    AND form_name = :form_name
+    """

scripts/REDCapDataExporter.py

@@ -0,0 +1,55 @@
+import logging
+import traceback
+from redcap import Project, RedcapError
+
+
+class REDCapDataExporter:
+    """Class that handles the exporting of RedCAP data"""
+
+    def __init__(self, base_url, api_token):
+        try:
+            self.project = Project(base_url, api_token)
+        except Exception as e:
+            logging.error(f"Error initializing REDCap project: {e}")
+            raise
+
+    def export_form(self, form_name, fields=None):
+        """Exports all patient's RedCap data by form name, and specific fields if desired, only if they are complete.
+
+        Args:
+            form_name (str): RedCap form name
+            fields (array[str], optional): Array of the name of the fields that we want to export from the form. Defaults to None.
+
+        Returns:
+            DataFrame: Pandas dataframe containing the desired form data, excluding the RedCap default fields 'redcap_repeat_instrument', 'redcap_repeat_instance', and 'formname_complete'
+        """
+        try:
+            df = self.project.export_records(
+                forms=[form_name],
+                format_type="df",
+                raw_or_label="raw",
+                filter_logic=f"[{form_name}_complete] = 2 or [{form_name}_complete] = 1",
+                df_kwargs={"dtype": "object", "usecols": fields},
+            )
+            fields_to_drop = [
+                "redcap_repeat_instrument",
+                "redcap_repeat_instance",
+                f"{form_name}_complete",
+            ]
+            return df.drop(columns=[col for col in fields_to_drop if col in df.columns])
+        except Exception:
+            logging.error(f"Error exporting form {form_name}: {traceback.format_exc()}")
+
+    def export_hospital_codes(self):
+        """Export RedCap DAG info. Given that each hospital has it's own unique DAG, this information is equivalent
+        to the hospital identifying codes.
+
+        Returns:
+            dict: Python dictionary containing the DAG names and it's unique RedCap ID.
+        """
+        try:
+            hospital_code_dict = self.project.export_dags()
+            return hospital_code_dict
+        except RedcapError as e:
+            print(f"Error fetching hospital codes: {e}")
+            return

scripts/RosettaParser.py

@@ -0,0 +1,65 @@
+import pandas as pd
+import logging
+
+
+class RosettaParser:
+    """Class that handles operations on the Rosetta ETL file"""
+
+    def __init__(self, rosetta_file):
+        try:
+            self.rosetta = pd.read_csv(rosetta_file, sep=";").fillna("NONE")
+        except Exception as e:
+            logging.error(f"Error loading rosetta file {rosetta_file}: {e}")
+            self.rosetta = pd.DataFrame()
+
+    def get_concept_id(self, variable, branch, value="NONE"):
+        """Get OMOP concept ID from the rosetta file by RedCap variable name, semantics branch name and value.
+
+        Args:
+            variable (str): Name of the RedCap variable that we want to get the OMOP concept id for.
+            branch (str): Name of the semantics branch associated to the OMOP concept id we want to get for the variable.
+            value (str, optional): Value of the RedCap vairable. Defaults to "NONE".
+
+        Raises:
+            ValueError: No matching concept id for the specific variable, value and branch input.
+
+        Returns:
+            str: OMOP concept id from the rosetta file.
+        """
+        df = self.rosetta[
+            (self.rosetta["variable"] == variable)
+            & (self.rosetta["source_value"] == value)
+            & (self.rosetta["branch"] == branch)
+        ]
+        if not df.empty:
+            return df["concept_id"].iloc[0]
+        else:
+            error_msg = f"No matching concept id for variable '{variable}', value '{value}', and branch '{branch}' inside the rosetta file."
+            logging.error(error_msg)
+            raise ValueError(error_msg)
+
+    def get_concept_code(self, variable, branch, value="NONE"):
+        """Get concept code from the rosetta file by RedCap variable name, semantics branch name and value.
+
+        Args:
+            variable (str): Name of the RedCap variable that we want to get the concept code for.
+            branch (str): Name of the semantics branch associated to the concept code we want to get for the variable.
+            value (str, optional): Value of the RedCap vairable. Defaults to "NONE".
+
+        Raises:
+            ValueError: No matching concept id for the specific variable, value and branch input.
+
+        Returns:
+            str: Concept code from the rosetta file.
+        """
+        df = self.rosetta[
+            (self.rosetta["variable"] == variable)
+            & (self.rosetta["source_value"] == value)
+            & (self.rosetta["branch"] == branch)
+        ]
+        if not df.empty:
+            return df["concept_code"].iloc[0]
+        else:
+            error_msg = f"No matching concept code for variable '{variable}', value '{value}', and branch '{branch}' inside the rosetta file."
+            logging.error(error_msg)
+            raise ValueError(error_msg)

scripts/atc2rxnorm.csv

@@ -0,0 +1,152 @@
+atc_code;atc_name;conceptId;rx_norm_conceptName
+J02AA01;AMFOTERICINA B;1717240;amphotericin B
+J01GB06;AMIKACINA;1790868;amikacin
+J01CA04;AMOXICILINA;1713332;amoxicillin
+J01CR02;"AMOXICILINA; CLAVULANICO ACIDO";40105046;amoxicillin / clavulanate Oral Tablet
+J01CA01;AMPICILINA;1717327;ampicillin
+J01CR01;"AMPICILINA; SULBACTAM";590408;Ampicillin / Sulbactam Oral Tablet
+J02AX06;ANIDULAFUNGINA;19026450;anidulafungin
+J01FA10;AZITROMICINA;1734104;azithromycin
+J01DF01;AZTREONAM;1715117;aztreonam
+J04AK05;BEDAQUILINA;43012518;bedaquiline
+J01CE01;BENCILPENICILINA;1728416;penicillin G
+J01CE08;BENCILPENICILINA-BENZATINA;1728416;penicillin G
+J04AB30;CAPREOMICINA;19026710;capreomycin
+J01CA03;CARBENICILINA;1740546;carbenicillin
+J01CA05;CARINDACILINA ;1740546;carbenicillin
+J02AX04;CASPOFUNGINA;1718054;caspofungin
+J01DB10;CEFACETRILO ;40798709;Cefacetrile
+J01DC04;CEFACLOR;1768849;cefaclor
+J01DB05;CEFADROXILO;1769535;cefadroxil
+J01DB01;CEFALEXINA;1786621;cephalexin
+J01DB02;CEFALORIDINA ;19052683;cephaloridine
+J01DB03;CEFALOTINA ;19086759;cephalothin
+J01DC03;CEFAMANDOL ;19070174;cefamandole
+J01DB08;CEFAPIRINA ;19086790;cephapirin
+J01DB04;CEFAZOLINA;1771162;cefazolin
+J01DD16;CEFDITORENO;1747005;cefditoren
+J01DE01;CEFEPIMA;1748975;cefepime
+J01DI04;CEFIDEROCOL;37498010;cefiderocol
+J01DD08;CEFIXIMA;1796435;cefixime
+J01DC09;CEFMETAZOL ;19072255;cefmetazole
+J01DC12;CEFMINOX;36860550;CEFMINOX
+J01DC06;CEFONICID;19072857;cefonicid
+J01DD12;CEFOPERAZONA ;1773402;cefoperazone
+J01DD62;CEFOPERAZONA + INHIBIDOR BETALACTAMASA;35136564;Cefoperazone / Sulbactam Injectable Solution
+J01DD01;CEFOTAXIMA;1774470;cefotaxime
+J01DC01;CEFOXITINA;1775741;cefoxitin
+J01DD13;CEFPODOXIMA;1749008;cefpodoxime
+J01DB09;CEFRADINA ;1786842;cephradine
+J01DI02;CEFTAROLINA;40230597;ceftaroline
+J01DD02;CEFTAZIDIMA;1776684;ceftazidime
+J01DD52;"CEFTAZIDIMA; AVIBACTAM";40745139;avibactam / Ceftazidime Injectable Solution
+J01DD14;CEFTIBUTENO ;1749083;ceftibuten
+J01DD07;CEFTIZOXIMA ;1777254;ceftizoxime
+J01DI01;CEFTOBIPROL;36878858;Ceftobiprole
+J01DI54;"CEFTOLOZANO; TAZOBACTAM";41298609;ceftolozane / tazobactam Injectable Solution
+J01DD04;CEFTRIAXONA;1777806;ceftriaxone
+J01DC02;CEFUROXIMA;1778162;cefuroxime
+J04AB01;CICLOSERINA;1710446;cycloserine
+J01MA02;CIPROFLOXACINO;1797513;ciprofloxacin
+J01FA09;CLARITROMICINA;1750500;clarithromycin
+J01FF01;CLINDAMICINA;997881;clindamycin
+J04BA01;CLOFAZIMINA;1798476;clofazimine
+J01BA01;CLORAMFENICOL ;990069;chloramphenicol
+J01AA03;CLORTETRACICLINA ;19095043;chlortetracycline
+J01CF02;CLOXACILINA;1800835;cloxacillin
+J01XB01;COLISTINA;901845;colistin
+J01EE01;COTRIMOXAZOL (SULFAMETOXAZOL + TRIMETOPRIMA);43763616;Sulfamethoxazole / Trimethoprim Oral Tablet [Cotrimoxazol Al]
+J01XA04;DALBAVANCINA;45774861;dalbavancin
+J01XX09;DAPTOMICINA;1786617;daptomycin
+J01MA23;DELAFLOXACINO;1592954;delafloxacin
+J04AK06;DELAMANID;36879076;Delamanid
+J01AA01;DEMECLOCICLINA ;1714527;demeclocycline
+J01GB09;DIBEKACINA ;19023508;dibekacin
+J01CF01;DICLOXACILINA ;1724666;dicloxacillin
+J01FA13;DIRITROMICINA ;1790024;dirithromycin
+J01AA02;DOXICICLINA;1738521;doxycycline
+J01MA04;ENOXACINO ;1743222;enoxacin
+J01AA13;ERAVACICLINA;35200469;eravacycline
+J01FA01;ERITROMICINA;1746940;erythromycin
+J01DH03;ERTAPENEM;1717963;ertapenem
+J01XX04;ESPECTINOMICINA ;1701651;spectinomycin
+J01FA02;ESPIRAMICINA;19070251;spiramycin
+J01RA04;"ESPIRAMICINA; METRONIDAZOL";40062184;metronidazole / spiramycin Oral Tablet
+J01GA01;ESTREPTOMICINA;1836191;streptomycin
+J04AK02;ETAMBUTOL;1749301;ethambutol
+J04AD03;ETIONAMIDA;1750074;ethionamide
+J01EB20;"FENAZOPIRIDINA; SULFAMETIZOL";40069477;phenazopyridine / sulfamethizole Oral Tablet
+J01CE02;FENOXIMETILPENICILINA;1729720;penicillin V
+J01CE10;FENOXIMETILPENICILINA-BENZATINA;1729720;penicillin V
+J01CF05;FLUCLOXACILINA ;19054936;floxacillin
+J02AC01;FLUCONAZOL;1754994;fluconazole
+J01XX01;FOSFOMICINA;956653;fosfomycin
+J01XC01;FUSIDATO;19010400;fusidate
+J01GB03;GENTAMICINA;45892419;gentamicin
+J01MA11;GREPAFLOXACINO ;1747032;grepafloxacin
+J01DH51;IMIPENEM;1778262;imipenem
+J01DH56;"IMIPENEM; RELEBACTAM";1361538;cilastatin / imipenem / relebactam Injection
+J02AC05;ISAVUCONAZOL SULFATO;46221284;isavuconazonium
+J04AC01;ISONIAZIDA;1782521;isoniazid
+J02AC02;ITRACONAZOL;1703653;itraconazole
+J01FA07;JOSAMICINA;19123240;josamycin
+J01GB04;KANAMICINA;1784749;kanamycin
+J01DD06;LATAMOXEF ;19126622;moxalactam
+J01MA12;LEVOFLOXACINO;1742253;levofloxacin
+J01AA04;LIMECICLINA;19092353;lymecycline
+J01FF02;LINCOMICINA;1790692;lincomycin
+J01XX08;LINEZOLID;1736887;linezolid
+J01CA11;MECILLINAM ;19123877;amdinocillin
+J01DH02;MEROPENEM;1709170;meropenem
+J01DH52;"MEROPENEM; VABORBACTAM";792534;meropenem / vaborbactam Injection
+J01CA14;METAMPICILINA ;19072054;methampicillin
+J01XD01;METRONIDAZOL;1707164;metronidazole
+J01CA10;MEZLOCILINA ;19007701;mezlocillin
+J02AX05;MICAFUNGINA SODICA;19018013;micafungin
+J01AA08;MINOCICLINA;1708880;minocycline
+J01FA11;MIOCAMICINA ;19009138;miocamycin
+J01MA14;MOXIFLOXACINO;1716903;moxifloxacin
+J01MB02;NALIDIXICO ACIDO ;986864;nalidixate
+J01GB07;NETILMICINA ;19017585;netilmicin
+J01XE01;NITROFURANTOINA;920293;nitrofurantoin
+J01XX07;NITROXOLINA;19015464;nitroxoline
+J01MA06;NORFLOXACINO;1721543;norfloxacin
+J01MA01;OFLOXACINO;923081;ofloxacin
+J01XA05;ORITAVANCINA;45776147;oritavancin
+J01AA06;OXITETRACICLINA ;925952;oxytetracycline
+J01MB05;OXOLINICO ACIDO ;19129642;oxolinic acid
+J01MA03;PEFLOXACINO ;19027679;pefloxacin
+J01RA01;PENICILINAS EN ASOCIACION CON OTROS ANTIBACTERIANOS ;41112508;Penicillin V Oral Solution [Antibiocin]
+J01MB04;PIPEMIDICO ACIDO;19010564;pipemidate
+J01CR05;"PIPERACILINA; TAZOBACTAM";46275426;piperacillin / tazobactam Injection
+J04AK01;PIRAZINAMIDA;1759455;pyrazinamide
+J01CA02;PIVAMPICILINA ;19047071;pivampicillin
+J01CA08;PIVMECILINAM;19088223;amdinocillin pivoxil
+J02AC04;POSACONAZOL;1704139;posaconazole
+J04AK08;PRETOMANID;37496482;pretomanid
+J01GB10;RIBOSTAMICINA ;36850424;RIBOSTAMYCIN
+J04AB04;RIFABUTINA;1777417;rifabutin
+J04AB02;RIFAMPICINA;1763204;rifampin
+J01FA06;ROXITROMICINA;19063874;roxithromycin
+J01GB08;SISOMICINA ;19136044;sisomicin
+J01CG01;SULBACTAM;1836241;sulbactam
+J01EC02;SULFADIAZINA;1836391;sulfadiazine
+J01ED02;SULFALENO ;19136423;sulfalene
+J01ED05;SULFAMETOXIPIRIDAZINA ;19000820;sulfamethoxypyridazine
+J01CR04;SULTAMICILINA ;43009009;sultamicillin
+J01CA15;TALAMPICILINA ;19002077;talampicillin
+J01XX11;TEDIZOLID;45775686;tedizolid
+J01XA02;TEICOPLANINA;19078399;teicoplanin
+J01XA03;TELAVANCINA;40166675;telavancin
+J01FA15;TELITROMICINA;1702911;telithromycin
+J01AA07;TETRACICLINA;1836948;tetracycline
+J01BA02;TIAMFENICOL ;19137362;thiamphenicol
+J01CA13;TICARCILINA ;1702364;ticarcillin
+J01AA12;TIGECICLINA;1742432;tigecycline
+J01XD02;TINIDAZOL ;1702559;tinidazole
+J01GB01;TOBRAMICINA;902722;tobramycin
+J01EA01;TRIMETOPRIMA;1705674;trimethoprim
+J01FA08;TROLEANDOMICINA ;19006043;troleandomycin
+J01MA13;TROVAFLOXACINO;1712549;trovafloxacin
+J01XA01;VANCOMICINA;1707687;vancomycin
+J02AC03;VORICONAZOL;1714277;voriconazole