Add EMP June 2025 data release

NEWS.md

@@ -1,6 +1,7 @@
 # zooper (development version)
 
--   Fixed issue where EMP macro amphipod counts from before 2014 were being converted to unidentified mysids instead of unidentified amphipods, since amphipods were not identified to species before 2014. 
+-   Added EMP data released in June 2025 (sampling from 1972–2023), **which no longer includes amphipods in the macro dataset**; see the [amphipods_notes (EDI 522)](https://portal.edirepository.org/nis/mapbrowse?scope=edi&identifier=522) for more information.
+-   ~~Fixed issue where EMP macro amphipod counts from before 2014 were being converted to unidentified mysids instead of unidentified amphipods, since amphipods were not identified to species before 2014.~~
 -   Fixed issue with biomass conversion for macro and micro zooplankton. Previously it was not taking into account the count and instead providing the BPUE if 1 individual were present. 
 -   Added the biomass for the surveys and size classes with the appropriate conversation factors and length data. This feature is still experimental
 -   Fixed issue with crosswalk that may impact Zoopsynther community output (values in the species column weren't consistent, some were genus species, some just species)

R/Download_utilities.R

@@ -76,17 +76,16 @@ zoop_urls<-function(Sources){
 
   if("EMP"%in%Sources){
     EMP_revision_url <- "https://pasta.lternet.edu/package/eml/edi/522"
-    #EMP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=EMP_revision_url, warn = FALSE), 1)
-    EMP_latest_revision = 11 #Keeping the old EMP dataset until I can go through and do a full update - next pull request.
+    EMP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=EMP_revision_url, warn = FALSE), 1)
     EMP_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/522/", EMP_latest_revision)
     EMP_entities <- Tryer(n=3, fun=readLines, con=EMP_pkg_url, warn = FALSE)
     EMP_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/522", EMP_latest_revision, EMP_entities, sep="/")
     names(EMP_entities) <- purrr::map_chr(EMP_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE))
 
-    out$EMP$Meso<-paste0(EMP_pkg_url, "/", EMP_entities["cb_matrix.csv"])
-    out$EMP$Micro<-paste0(EMP_pkg_url, "/", EMP_entities["pump_matrix.csv"])
-    out$EMP$Macro<-paste0(EMP_pkg_url, "/", EMP_entities["macro_matrix.csv"])
-    out$EMP$Lengths<-paste0(EMP_pkg_url, "/", EMP_entities["macro_lengths.csv"])
+    out$EMP$Meso<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2023_CBMatrix_EDI"])
+    out$EMP$Micro<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2023_PumpMatrix_EDI"])
+    out$EMP$Macro<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2023_MysidMatrix_EDI"])
+    out$EMP$Lengths<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2023_Mysid_Length_Data_EDI"])
 
   }
 

R/zoop_downloader.R

@@ -122,9 +122,9 @@ Zoopdownloader <- function(
                                                              OITHJUV="d", OTHCYCJUV="d", COPNAUP="d",
                                                              EURYNAUP="d", OTHCOPNAUP="d", PDIAPNAUP="d",
                                                              SINONAUP="d", BOSMINA="d", DAPHNIA="d",
-                                                             DIAPHAN="d",OTHCLADO="d", ASPLANCH="d",
-                                                             KERATELA="d",OTHROT="d", POLYARTH="d",
-                                                             SYNCH="d",SYNCHBIC="d", TRICHO="d",
+                                                             DIAPHAN="d", OTHCLADO="d", ASPLANCH="d",
+                                                             KERATELA="d", OTHROT="d", POLYARTH="d",
+                                                             SYNCH="d", SYNCHBIC="d", TRICHO="d",
                                                              BARNNAUP="d", CRABZOEA="d"))
 
     # Tranform from "wide" to "long" format, add some variables,
@@ -890,15 +890,11 @@ Zoopdownloader <- function(
                                    col_types=readr::cols_only(SampleDate="c", Time="c", StationNZ="c",
                                                               Chl_a="d", Secchi="d", Temperature="d",
                                                               ECSurfacePreTow="d", ECBottomPreTow="d",
-                                                              Volume="d", Depth="d", AmphipodCode="c", A_aspera="d",
-                                                              A_hwanhaiensis="d", A_macropsis="d", D_holmquistae="d",
-                                                              H_longirostris="d", N_kadiakensis="d", N_mercedis="d",
-                                                              Unidentified_mysid="d", A_spinicorne="d", A_stimpsoni="d",
-                                                              A_abdita="d", Ampithoe_sp="d", Caprelidae_sp="d",
-                                                              C_alienense="d", Crangonyx_sp="d", G_daiberi="d",
-                                                              G_japonica="d", Hyalella_sp="d", Monocorophium_sp="d",
-                                                              Oedicerotidae_sp="d", Pleustidae="d", Unidentified_Amphipod="d",
-                                                              Unidentified_Corophium="d", Unidentified_Gammarus="d", Amphipod_Total="d"))%>%
+                                                              Volume="d", Depth="d", `Acanthomysis aspera`="d",
+                                                              `Acanthomysis hwanhaiensis`="d", `Alienacanthomysis macropsis`="d",
+                                                              `Deltamysis holmquistae`="d", `Hyperacanthomysis longirostris`="d",
+                                                              `Neomysis kadiakensis`="d", `Neomysis mercedis`="d",
+                                                              `Unidentified Mysid`="d"))%>%
       dplyr::filter(dplyr::if_any(dplyr::everything(), ~ !is.na(.)))
 
     # Tranform from "wide" to "long" format, add some variables,
@@ -908,16 +904,15 @@ Zoopdownloader <- function(
       dplyr::mutate(SampleDate=lubridate::parse_date_time(.data$SampleDate, "%m/%d/%Y", tz="America/Los_Angeles"),
                     Datetime=lubridate::parse_date_time(dplyr::if_else(is.na(.data$Time), NA_character_, paste(.data$SampleDate, .data$Time)),
                                                         c("%Y-%m-%d %I:%M %p"), tz="Etc/GMT+8"), #create a variable for datetime
-                    Datetime=lubridate::with_tz(.data$Datetime, "America/Los_Angeles"), # Ensure everything ends up in local time
-                    Unidentified_Amphipod=dplyr::if_else(lubridate::year(.data$SampleDate)<2014, .data$Amphipod_Total, .data$Unidentified_Amphipod))%>% # Transfer pre 2014 amphipod counts to Amphipod_total
+                    Datetime=lubridate::with_tz(.data$Datetime, "America/Los_Angeles"))%>% # Ensure everything ends up in local time
       tidyr::pivot_longer(cols=c(-"SampleDate", -"Time", -"Datetime", -"StationNZ", -"Secchi", -"Chl_a", -"Temperature",
-                                 -"ECSurfacePreTow", -"ECBottomPreTow", -"Volume", -"Depth", -"AmphipodCode"),
+                                 -"ECSurfacePreTow", -"ECBottomPreTow", -"Volume", -"Depth"),
                           names_to="EMP_Macro", values_to="CPUE")%>% #transform from wide to long
       dplyr::mutate(Source="EMP",
                     SizeClass="Macro")%>% #add variable for data source
       dplyr::select("Source", Date = "SampleDate", "Datetime", Station="StationNZ", Chl = "Chl_a",
                     CondBott = "ECBottomPreTow", CondSurf = "ECSurfacePreTow", "Secchi", "SizeClass",
-                    "Temperature", BottomDepth="Depth", "Volume", "AmphipodCode", "EMP_Macro", "CPUE")%>% #Select for columns in common and rename columns to match
+                    "Temperature", BottomDepth="Depth", "Volume", "EMP_Macro", "CPUE")%>% #Select for columns in common and rename columns to match
       dplyr::left_join(Crosswalk%>% #Add in Taxnames, Lifestage, and taxonomic info
                          dplyr::select("EMP_Macro", "Lifestage", "Taxname", "Phylum", "Class",
                                        "Order", "Family", "Genus", "Species", "Intro", "EMPstart", "EMPend")%>% #only retain EMP codes
@@ -935,8 +930,7 @@ Zoopdownloader <- function(
         .data$CPUE==0 & .data$Date < .data$Intro ~ 0,
         .data$CPUE==0 & .data$Date >= .data$Intro & .data$Date < .data$EMPstart ~ NA_real_,
         .data$CPUE==0 & .data$Date >= .data$EMPstart & .data$Date < .data$EMPend ~ 0,
-        .data$CPUE==0 & .data$Date >= .data$EMPend ~ NA_real_),
-        CPUE=dplyr::if_else(.data$AmphipodCode!="A" & .data$Order=="Amphipoda", NA_real_, .data$CPUE))%>% # Remove any tainted amphipod data (e.g., veg in net)
+        .data$CPUE==0 & .data$Date >= .data$EMPend ~ NA_real_))%>%
       dplyr::select(-"EMP_Macro", -"EMPstart", -"EMPend", -"Intro")%>% #Remove EMP taxa codes
       dtplyr::lazy_dt()%>% #Speed up code using dtplyr package that takes advantage of data.table speed
       dplyr::group_by(dplyr::across(-"CPUE"))%>%
@@ -955,9 +949,9 @@ Zoopdownloader <- function(
 
         lengths.list[["EMP_Lengths"]]<-readr::read_csv(file.path(Data_folder, "EMP_Lengths.csv"),
                                                        col_types=readr::cols_only(SampleDate="c", StationNZ="c",
-                                                                                  SpeciesName="c", Size="d", AdjustedFreq="d"))%>%
+                                                                                  SpeciesName="c", Length="d", AdjustedFreq="d"))%>%
           dplyr::mutate(SampleDate=lubridate::parse_date_time(.data$SampleDate, "%m/%d/%Y", tz="America/Los_Angeles"))%>%
-          dplyr::rename(Date="SampleDate", Station="StationNZ", EMP_Lengths="SpeciesName", Length="Size", Count="AdjustedFreq")%>%
+          dplyr::rename(Date="SampleDate", Station="StationNZ", EMP_Lengths="SpeciesName", Length="Length", Count="AdjustedFreq")%>%
           dplyr::left_join(Crosswalk%>% #Add in Taxnames, Lifestage, and taxonomic info
                              dplyr::select("EMP_Lengths", "Lifestage", "Taxname")%>% #only retain EMP codes
                              dplyr::filter(!is.na(.data$EMP_Lengths))%>% #Only retain Taxnames corresponding to EMP codes