The total size of the 10 globals exported for future expression

[trafficonese]

I saw this issue in one of my Shiny logs, and found similar ones here: satijalab/seurat#1845
But I am not using the seurat package, but the AsyncProgress function from ipc.

Loading required package: shiny

Listening on http://0.0.0.0:9999
Unhandled promise error: ungültiger Argumenttyp
Warnung: Error in getGlobalsAndPackages: The total size of the 10 globals exported for future expression (...
This exceeds the maximum allowed size of 500.00 MiB (option 'future.globals.maxSize'). The three largest globals are ‘progress’ (507.65 MiB of class ‘environment’), ‘dbConnect’ (27.67 KiB of class ‘function’) and ‘svd_changes’ (8.24 KiB of class ‘list’)

This R snippet trigered that error, but I was mainly wondering why the progress object should be around 507 MiB..

progress = AsyncProgress$new(message="Changes saved")

myFuture <- future(seed = NULL, label = "SaveToDatabase", {
  con_fut <- dbConnect(...)
  on.exit(dbDisconnect(con_fut), add = TRUE)
  progress$inc(0.1, message = "Connected")
  handle_svd_changes(con_fut, svd_changes, name, config, tblname, progress)
})
then(
  myFuture,
  onFulfilled = function(value) {
    progress$close()
  },
  onRejected = function(reason) {
    progress$close()
  }
)

I am using this option now as commented n the other issue:
options(future.globals.maxSize = 8000 * 1024^2)
We'll see if it reappears..

[ifellows]

I do not know why or how progress could get large like that. Can you please provide a runnable self-contained example where the issue occurs?

[trafficonese]

I am seeing a new issue ( Error in saveRDS ) for the code above and openend an issue here: r-lib/callr#292
but maybe it belongs in another package: ipc, future.callr, callr??
Any help in debugging ths bug would be highly appreciated!

[gergness]

Hi, I'm also hitting this (well I'm not quite at an error yet, but I'm noticing that as my app gets more complex, it gets slower and slower to start subprocesses). I think it is related to how R6 objects have to pass everything in their environment, and in this case it includes the shiny::session object. Here's an R6 issue talking about this: r-lib/R6#157

Reading through source code, it doesn't actually seem like the worker process needs access to the Progress object (because doesn't producer$fireEval just capture the expression that should be run and save it to text?), but my attempts to pass just the source and recreate the functions needed to update the progress hasn't worked yet.

Here's a demo app and video showing what is happening.

Screen.Recording.2024-12-10.at.12.08.10.PM.mov

library(shiny)
library(ipc)
library(future)
library(promises)
library(lobstr)
plan(multisession)


# Define UI for application that draws a histogram
ui <- fluidPage(
    
    # Application title
    titlePanel("Async With Progress Bar"),
    
    # Sidebar with a slider input for number of bins
    sidebarLayout(
        sidebarPanel(
            tags$div(
                actionButton('run', 'Run'),
                actionButton('cancel', 'Cancel')
            ),
            tags$br(),
            tags$div(
                actionButton('add', 'Add Data to Session'),
                tags$p("Simulate an increasingly complex shiny app, adding ~100Mb to the session object per click"),
                verbatimTextOutput("current_session_size")
            )
        ),
        
        # Show a plot of the generated distribution
        mainPanel(
            tableOutput("result"),
            tags$h2("Sizes"),
            tableOutput("sizes_table")
        )
    )
)

server <- function(input, output, session) {
    object_sizes <- reactiveVal(NULL)
    N <- 5
    
    interruptor <- AsyncInterruptor$new()    # To signal STOP to the future
    
    result_val <- reactiveVal()
    running <- reactiveVal(FALSE)
    observeEvent(input$run,{
        
        #Don't do anything if in the middle of a run
        if(running())
            return(NULL)
        running(TRUE)
        
        # Create new progress bar
        progress <- AsyncProgress$new(message="Complex analysis")
        
        object_sizes(
            rbind(object_sizes(), data.frame(
                session = format(obj_size(session)),
                progress = format(obj_size(progress)),
                queue = format(obj_size(progress$.__enclos_env__$private$queue)),
                source = format(obj_size(progress$.__enclos_env__$private$queue$consumer$.__enclos_env__$private$source))
            ))
        )
        
        result_val(NULL)
        
        
        fut <- future({
            for(i in 1:N){
                # Some important computation
                Sys.sleep(.2)
                
                # Increment progress bar
                progress$inc(1/N)
                
                # throw errors that were signal (if Cancel was clicked)
                interruptor$execInterrupts()
            }
            
            data.frame(result="Insightful analysis")
        }) %...>% result_val
        
        # Show notification on error or user interrupt
        fut <- catch(fut,
                     function(e){
                         result_val(NULL)
                         print(e$message)
                         showNotification(e$message)
                     })
        
        # When done with analysis, remove progress bar
        fut <- finally(fut, function(){
            progress$close()
            running(FALSE) # Declare done with run
        })
        
        # Return something other than the future so we don't block the UI
        NULL
    })
    
    
    # Send interrupt signal to future
    observeEvent(input$cancel,{
        if(running())
            interruptor$interrupt("User Interrupt")
    })
    
    # I'm not exactly sure what in shiny makes the session grow, but it can
    # This is just a way to force it to happen
    observeEvent(input$add, {
        session$userData$stuff <- c(
            session$userData$stuff,
            list(data.frame(
                x = rnorm(6e6),
                z = sample(letters, 6e6, TRUE)
            ))
        )
        cur_session_size(format(obj_size(session)))
    })
    cur_session_size <- reactiveVal(format(obj_size(session)))
    
    
    output$result <- renderTable({
        req(result_val())
    })
    
    output$sizes_table <- renderTable({req(object_sizes())})
    output$current_session_size <- renderText({
        input$add
        paste0("Current session size: ", cur_session_size())
    })
}

# Run the application
shinyApp(ui = ui, server = server)

[gergness]

Actually was able to get it to work, but wonder if you have any better ideas:

passing the source and using it directly:

        source <- progress$.__enclos_env__$private$queue$consumer$.__enclos_env__$private$source

...

                # Increment progress bar
                # ---- Mock the `progress$inc` function ----
                # progress$inc(1/N)
                q <- queue(source)
                q$producer$fireEval({
                    do.call(private$progress$inc, args)
                }, list(args = list(amount = 1/N)))
                # ------------------------------------------

library(shiny)
library(ipc)
library(future)
library(promises)
library(lobstr)
plan(multisession)


# Define UI for application that draws a histogram
ui <- fluidPage(
    
    # Application title
    titlePanel("Async With Progress Bar"),
    
    # Sidebar with a slider input for number of bins
    sidebarLayout(
        sidebarPanel(
            tags$div(
                actionButton('run', 'Run')
            ),
            tags$br(),
            tags$div(
                actionButton('add', 'Add Data to Session'),
                tags$p("Simulate an increasingly complex shiny app, adding ~600Mb to the session object per click"),
                verbatimTextOutput("current_session_size")
            )
        ),
        
        # Show a plot of the generated distribution
        mainPanel(
            tableOutput("result"),
            tags$h2("Sizes"),
            tableOutput("sizes_table")
        )
    )
)

server <- function(input, output, session) {
    object_sizes <- reactiveVal(NULL)
    N <- 5

    result_val <- reactiveVal()
    running <- reactiveVal(FALSE)
    observeEvent(input$run,{
        
        #Don't do anything if in the middle of a run
        if(running())
            return(NULL)
        running(TRUE)
        
        # Create new progress bar
        progress <- AsyncProgress$new(message="Complex analysis")
        
        object_sizes(
            rbind(object_sizes(), data.frame(
                session = format(obj_size(session)),
                progress = format(obj_size(progress)),
                queue = format(obj_size(progress$.__enclos_env__$private$queue)),
                source = format(obj_size(progress$.__enclos_env__$private$queue$consumer$.__enclos_env__$private$source))
            ))
        )
        source <- progress$.__enclos_env__$private$queue$consumer$.__enclos_env__$private$source
        
        result_val(NULL)
        
        
        fut <- future({
            for(i in 1:N){
                # Some important computation
                Sys.sleep(.2)
                
                # Increment progress bar
                # ---- Mock the `progress$inc` function ----
                # progress$inc(1/N)
                q <- queue(source)
                q$producer$fireEval({
                    do.call(private$progress$inc, args)
                }, list(args = list(amount = 1/N)))
                # ------------------------------------------
            }
            
            data.frame(result="Insightful analysis")
        }) %...>% result_val
        
        # Show notification on error or user interrupt
        fut <- catch(fut,
                     function(e){
                         result_val(NULL)
                         print(e$message)
                         showNotification(e$message)
                     })
        
        # When done with analysis, remove progress bar
        fut <- finally(fut, function(){
            progress$close()
            running(FALSE) # Declare done with run
        })
        
        # Return something other than the future so we don't block the UI
        NULL
    })
    
    
    
    # I'm not exactly sure what in shiny makes the session grow, but it can
    # This is just a way to force it to happen
    observeEvent(input$add, {
        session$userData$stuff <- c(
            session$userData$stuff,
            list(data.frame(
                x = rnorm(4e7),
                z = sample(letters, 4e7, TRUE)
            ))
        )
        cur_session_size(format(obj_size(session)))
    })
    cur_session_size <- reactiveVal(format(obj_size(session)))
    
    
    output$result <- renderTable({
        req(result_val())
    })
    
    output$sizes_table <- renderTable({req(object_sizes())})
    output$current_session_size <- renderText({
        input$add
        paste0("Current session size: ", cur_session_size())
    })
}

# Run the application
shinyApp(ui = ui, server = server)