data source and citation table, IMPORTANT notes below

global.R

@@ -54,16 +54,27 @@ options(dplyr.summarise.inform = FALSE,
 conf <- jsonlite::fromJSON("config.json")
 
 ## uncomment and run this (without saving script) to create initial connection to our shinyapps account
-# rsconnect::setAccountInfo(name = 'cuahsi',
-#                           token = conf$shinyapps_cuahsi_token,
-#                           secret = conf$shinyapps_cuahsi_secret)
+## rsconnect::setAccountInfo(name = 'cuahsi',
+##                           token = conf$shinyapps_cuahsi_token,
+##                           secret = conf$shinyapps_cuahsi_secret)
 
 ## uncomment and run this (without saving script) to deploy app. you may need to establish your own path
-# options(rsconnect.http.timeout = 50000)
-# options(rsconnect.max.bundle.size = 8 * 1024 * 1024 * 1024)
-# rsconnect::deployApp('~/science/macrosheds/portal',
-#                      appName = 'macrosheds',
-#                      account = 'cuahsi')
+## options(rsconnect.http.timeout = 10000000)
+## options(rsconnect.max.bundle.size = 8 * 1024 * 1024 * 1024)
+## while(TRUE){
+##     catch = 0
+##     catch_counter = 0
+
+##     catch = try({
+##         catch_counter = catch_counter + 1
+##         print(paste('MacroSheds deploy attempt, tries on this run:', catch_counter))
+##         rsconnect::deployApp('~/science/macrosheds/portal',
+##                              appName = 'macrosheds',
+##                              account = 'cuahsi',
+##                              forceUpdate = TRUE)
+##     })
+##     if(! inherits(catch, 'try-error')) break
+## }
 
 # rsconnect::showLogs(appName = "macrosheds", streaming = TRUE)
 

helpers.R

@@ -1100,7 +1100,6 @@ load_portal_config <- function(from_where) {
             col_types = 'ccccccccnnnnnccccc'
         ))
 
-
         # universal_products <- sm(googlesheets4::read_sheet(
         #     conf$univ_prods_gsheet,
         #     na = c('', 'NA'),
@@ -1120,13 +1119,27 @@ load_portal_config <- function(from_where) {
             col_types = 'c'
         ))
 
+        detection_limits <- sm(googlesheets4::read_sheet(
+            conf$detection_limits_gsheet,
+            na = c('', 'NA'),
+            col_types = 'c'
+        ))
+
+        products_registry <- sm(googlesheets4::read_sheet(
+            conf$product_registry_gsheet,
+            na = c('', 'NA'),
+            col_types = 'c'
+        ))
+
     } else if (from_where == "local") {
 
         variables <- sm(read_csv("data/general/variables.csv"))
         site_data <- sm(read_csv("data/general/site_data.csv"))
         # universal_products <- sm(read_csv("data/general/universal_products.csv"))
         disturbance_record <- sm(read_csv("data/general/disturbance_record.csv"))
         site_doi_license <- sm(read_csv("data/general/site_doi_license.csv"))
+        detection_limits <- sm(read_csv("data/general/detection_limits.csv"))
+        products_registry <- sm(read_csv("data/general/products_registry.csv"))
 
     } else {
         stop('from_where must be either "local" or "remote"')
@@ -1151,6 +1164,22 @@ load_portal_config <- function(from_where) {
         site_doi_license,
         pos = .GlobalEnv
     )
+
+    assign("detection_limits",
+        detection_limits,
+        pos = .GlobalEnv
+    )
+
+    assign("products_registry",
+        products_registry,
+        pos = .GlobalEnv
+    )
+
+    citation_table_data <- create_prodname_from_prodcode_table()
+    assign("citation_table_data",
+        citation_table_data,
+        pos = .GlobalEnv
+    )
 }
 
 generate_dropdown_varlist_ws <- function(variables) {
@@ -1483,3 +1512,49 @@ read_combine_ws_traits <- function(ws_prod, ws_var, new_var_name, dmns,
     }
     return(combined_data)
 }
+
+create_prodname_from_prodcode_table <- function() {
+
+    citation_table_data <- site_doi_license %>%
+      left_join(products_registry, by = c('domain', "macrosheds_prodcode" = 'prodcode')) %>%
+      relocate(prodname, .after = domain) %>%
+      rename(data_product_type = prodname) %>%
+      select(-retrieve_status, -munge_status, -derive_status, -components, -component, -NA_status, -retrieve_status.1, -retrieve_status.2)
+
+    return(citation_table_data)
+}
+
+create_product_registry <- function(ms_root) {
+    src_root = file.path(ms_root, 'src')
+    src_nwks = list.dirs(src_root, recursive = F)
+    src_nwks = src_nwks[!grepl('output|dev', src_nwks)]
+
+    for(nwk in src_nwks) {
+        src_dmns = list.dirs(nwk, recursive = F)
+
+        for(dmn in src_dmns) {
+            src_dmn_files <- list.files(dmn, recursive = F, full.names = TRUE)
+            products <- try(
+                read.csv(src_dmn_files[grepl('products.csv', src_dmn_files)])
+            )
+
+            if(inherits(products, 'try-error')) {
+                next
+            } else {
+                dmn_name = str_extract(dmn, '([^\\/]+$)')
+                products$domain = dmn_name
+            }
+
+            if(!exists('products_registry')) {
+                products_registry = products
+            } else {
+                products_registry = dplyr::bind_rows(products_registry, products)
+            }
+        }
+    }
+
+    products_registry <- products_registry %>%
+      relocate(domain, .before = prodcode)
+
+    googlesheets4::write_sheet(products_registry, conf$product_registry_gsheet, sheet = 'products_registry')
+}

server/nSiteNVar_server.R

@@ -1984,3 +1984,6 @@ outputOptions(output,
     suspendWhenHidden = FALSE,
     priority = 100
 )
+
+output$citation_table_ts = DT::renderDataTable(citation_table_data, selection = 'single',
+                            rownames = FALSE, options = list(scrollX = 'true'))

server/summary_biplot_server.R

@@ -1197,3 +1197,6 @@ timeSliderChanged <- eventReactive(
     }
 ) %>%
     debounce(1000)
+
+output$citation_table = DT::renderDataTable(citation_table_data, selection = 'single',
+                            rownames = FALSE, options = list(scrollX = 'true'))

ui/landing_slides.html

@@ -12,6 +12,9 @@
                      </div>
                 </div> -->
             <br>
+            <em>this portal may display sites not yet included in the published <a href="https://portal.edirepository.org/nis/mapbrowse?scope=edi&identifier=1262&revision=1">MacroSheds dataset</a>.</em>
+            <br/>
+            <br/>
             <strong>Watershed ecosystem science has identified plenty of idiosyncrasy within watersheds,
                 but hasn't produced many general theories about watersheds at large. A major reason has
                 been the challenge of data access and integration across all the organizations that house watershed data.

ui/nSiteNVar_ui.R

@@ -13,7 +13,6 @@ nSiteNVar_tab <- tabPanel("Time Series",
             br(),
             div(
                 style = "display: none",
-
                 # these are ineligant alternatives to an "invalidateInput" function
                 # that must exist to accompany updateSelectizeInput, etc., but
                 # seemingly doesn't
@@ -106,10 +105,11 @@ nSiteNVar_tab <- tabPanel("Time Series",
                 ),
             ),
             br(),
-            div("Unit", class = "widget-title text-center"),
+            div("Unit", class = "widget-title text-center", title = "solute flux and volume weighted concentration data are both calculated by MacroSheds, and not from source data. See Notes tab for information on these calculations"),
             div("(Applies to solutes only)",
                 class = "widget-caption text-center"
             ),
+            br(),
             div(
                 id = "cf_div",
                 radioButtons("CONC_FLUX3",
@@ -363,9 +363,24 @@ nSiteNVar_tab <- tabPanel("Time Series",
                     HTML(paste0(
                         '<p style="color: blue; font-size: 0.75em">',
                         "Click and drag to zoom. Double-click to reset view.</p>"
-                    ))
+                    )),
+                    actionButton("showTable", " data source and citation information", icon = icon("table", style = "color: #d88546"), title="information available in this table includes citation, license, data source url, data source contact, and much more metadata for each data product in MacroSheds"),
+                    bsModal("modalExample", "Data Table", "showTable", size = "large",
+                            dataTableOutput("citation_table_ts"))
                 )
-            )
+            ),
+
+            ## fluidRow(
+            ##     column(3),
+            ##     column(4,
+            ##             br(),
+            ##             actionButton("showTable", "macrosheds data source and citation information", icon = icon("table")),
+            ##             br()
+            ##         ),
+            ##     column(5)
+            ## ),
+            ## bsModal("modalExample", "Data Table", "showTable", size = "large",
+            ##         dataTableOutput("citation_table"))
         )
     )
 )

ui/notes_tab.html

@@ -137,7 +137,7 @@ <h4 class="panel-title">
         <div class="panel-heading">
           <h4 class="panel-title">
             <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#collapseFive">
-              Volume Weighted Concentration
+              Volume Weighted Concentration and Flux
             </a>
           </h4>
         </div>
@@ -154,6 +154,9 @@ <h4 class="panel-title">
                 <li>
                     Also <strong>NOTE</strong>, that for efficiency in computing VWC, we do not remove measurements from the denominator that are unmatched in the numerator (e.g. timepoints with a measured Q value, but no calculated F value). For some sites and time periods, this will produce a weighted mean (VWC) that is skewed. Such a mismatch is uncommon, and should rarely be of great consequence. Still, before you draw any conclusions from a VWC trend, be sure to switch the view back to "Daily" aggregation and verify that the trend is real, not just an artifact of there being more Q or precipitation measurements than there are flux estimates. In particular, this might be relevant if there was a large storm during a certain month, and that storm was captured in the discharge series, but not in the flux series.
                 </li>
+                <li>
+                  <strong> flux </strong> is calculated by the formula <code> F = Q * C </code> where Q is discharge and C is concentration of a solute. This is the only calculation for flux available on the MacroSheds portal, and all such calculations are made by MacroSheds and are not in any way the official flux values of the original data source, and it must be noted that many data sources report their own flux values, often using more advanced flux estimation methods which have been chosen and developed for their specific needs. MacroSheds currently uses the most basic possible flux calculation method for simplicity and transparency.
+                </li>
             </ul>
           </div>
         </div>
@@ -191,12 +194,16 @@ <h4 class="panel-title">
             <ul>
                 <li>
                     All MacroSheds data are from U.S. federally funded projects,
-                    <em>except</em> data from the Krycklan Catchment Study (Sweden).
+                    <em>except</em> data from the Krycklan Catchment Study (Sweden), and data from the <a href = "https://metrocouncil.org/About-Us/What-We-Do/Departments/Environmental-Services.aspx"> Metropolitan Council Environmental Services </a> (Minneapolis and St. Paul, Minnesota -- pre-release).
+                </li>
+                <li>It is encouraged that users "insert MCES request here" if you plan to use MCES data, especially for any uses intended for publication. To learn more about our data sources -- including licensing and citation information, original dataset links, contact information, and more -- click on the "macrosheds data source and citation information" button below the data visualizations on either the time series or site comparison tabs.
                 </li>
                 <li>
                     MacroSheds cannot retain all source metadata, so please consult primary source documentation
                     for specifics on sampling/analytical methods, quality control, and sampling sites.
                 </li>
+                <li>Note that MacroSheds versions of source data may deviate from original data by filtering and aggregations, as well as include additional products (such as flux) which may not be included or deviate from similar products in original data sources. Data in MacroSheds is not represent in any way an official dataset of any source, and is entirely a seperate entity. Any discrepancies should be resolved by defering to the expertise and data version of the dataset source.
+                </li>
             </ul>
           </div>
         </div>

ui/summary_biplot_ui.R

@@ -164,7 +164,18 @@ summary_biplot_tab <- tabPanel("Site Comparison",
                 value = c(dtrng[1], dtrng[2]),
                 width = "100%", timeFormat = "%Y", step = 365,
                 dragRange = TRUE
-            )
+                ),
+          fluidRow(
+            column(3),
+            column(4,
+                    br(),
+                    actionButton("showTable", " data source and citation information", icon = icon("table", style = "color: #d88546"), title="information available in this table includes citation, license, data source url, data source contact, and much more metadata for each data product in MacroSheds"),
+                    br()
+                ),
+            column(5)
+          ),
+            bsModal("modalExample", "Data Table", "showTable", size = "large",
+                    dataTableOutput("citation_table"))
         )
     )
 )